{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12/25/2019 17:48:15 - INFO - utils -   device: cuda n_gpu: 1\n",
      "12/25/2019 17:48:15 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file /data/disk1/sharing/pretrained_embedding/bert/uncased_L-12_H-768_A-12/vocab.txt\n",
      "12/25/2019 17:48:15 - INFO - pytorch_pretrained_bert.modeling -   loading archive file /data/disk1/sharing/pretrained_embedding/bert/uncased_L-12_H-768_A-12\n",
      "12/25/2019 17:48:15 - INFO - pytorch_pretrained_bert.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "12/25/2019 17:48:18 - INFO - pytorch_pretrained_bert.modeling -   Weights of BertForConstrainClustering not initialized from pretrained model: ['cluster_layer', 'dense.weight', 'dense.bias', 'classifier.weight', 'classifier.bias']\n",
      "12/25/2019 17:48:18 - INFO - pytorch_pretrained_bert.modeling -   Weights from pretrained model not used in BertForConstrainClustering: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
     ]
    }
   ],
   "source": [
    "from model import BertForConstrainClustering\n",
    "from utils import *\n",
    "import argparse\n",
    "import random\n",
    "import torch\n",
    "import os\n",
    "import pandas as pd\n",
    "from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset)\n",
    "from tqdm import trange\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "from pytorch_pretrained_bert.optimization import BertAdam, warmup_linear\n",
    "from pytorch_pretrained_bert.modeling import WEIGHTS_NAME, CONFIG_NAME\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from datetime import datetime\n",
    "import warnings\n",
    "warnings.warn = warn\n",
    "\n",
    "results_all = {}\n",
    "seed = 44\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "task_name = dataset = 'snips'\n",
    "fraction = 1\n",
    "labeled_ratio = 0.1\n",
    "unknown_cls_ratio = 0.25\n",
    "\n",
    "data_dir = 'data/' + task_name\n",
    "output_dir = '/tmp/' + datetime.now().strftime('%Y-%m-%d_%H:%M:%S')\n",
    "bert_model = \"/data/disk1/sharing/pretrained_embedding/bert/uncased_L-12_H-768_A-12\"\n",
    "num_train_epochs = 46\n",
    "\n",
    "max_seq_task = {\n",
    "    \"snips\": 35,\n",
    "    'dbpedia': 54,\n",
    "    \"stackoverflow\": 20,\n",
    "}\n",
    "max_seq_length = max_seq_task[task_name]\n",
    "train_batch_size = 256\n",
    "eval_batch_size = 256\n",
    "learning_rate = 2e-5\n",
    "warmup_proportion = 0.1\n",
    "\n",
    "processors = {\n",
    "    \"snips\": SnipsProcessor,\n",
    "    'dbpedia': Dbpedia_Processor,\n",
    "    \"stackoverflow\": Stackoverflow_Processor,\n",
    "}\n",
    "\n",
    "num_labels_task = {\n",
    "    \"snips\": 7,\n",
    "    'dbpedia': 14,\n",
    "    \"stackoverflow\": 20,\n",
    "}\n",
    "\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "logger.info(\"device: {} n_gpu: {}\".format(device, n_gpu))\n",
    "logger.disabled = True\n",
    "\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "if os.path.exists(output_dir) and os.listdir(output_dir):\n",
    "    raise ValueError(\"Output directory ({}) already exists and is not empty.\".format(output_dir))\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)\n",
    "\n",
    "if task_name not in processors:\n",
    "    raise ValueError(\"Task not found: %s\" % (task_name))\n",
    "\n",
    "processor = processors[task_name]()\n",
    "num_labels = num_labels_task[task_name]\n",
    "label_list = processor.get_labels()\n",
    "tokenizer = BertTokenizer.from_pretrained(bert_model, do_lower_case=True)\n",
    "\n",
    "# Freezing all transformer (except the last layer)\n",
    "model = BertForConstrainClustering.from_pretrained(bert_model, num_labels = num_labels*fraction)\n",
    "for name, param in model.bert.named_parameters():  \n",
    "    param.requires_grad = False\n",
    "    if \"encoder.layer.11\" in name or \"pooler\" in name:\n",
    "        param.requires_grad = True\n",
    "model.to(device)\n",
    "\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "]\n",
    "\n",
    "train_examples = processor.get_train_examples(data_dir)\n",
    "num_train_optimization_steps = int(len(train_examples) / train_batch_size) * num_train_epochs\n",
    "optimizer = BertAdam(optimizer_grouped_parameters,\n",
    "                     lr=learning_rate,\n",
    "                     warmup=warmup_proportion,\n",
    "                     t_total=num_train_optimization_steps)\n",
    "\n",
    "## Settings of unknown classes discovery\n",
    "# 1. Select 25% classes as unknown(-1)\n",
    "# 2. Set 90% of examples as unknown(-1)\n",
    "n_unknown_cls = round(num_labels*unknown_cls_ratio)\n",
    "label_unknown = np.random.choice(np.array(label_list), n_unknown_cls, replace=False)\n",
    "\n",
    "train_labeled_examples, train_unlabeled_examples = [], []\n",
    "for example in train_examples:\n",
    "    if (example.label not in label_unknown) and (np.random.uniform(0, 1) <= labeled_ratio):\n",
    "        train_labeled_examples.append(example)\n",
    "    else:\n",
    "        train_unlabeled_examples.append(example)\n",
    "\n",
    "\n",
    "train_loss = 0\n",
    "train_features = convert_examples_to_features(train_examples, label_list, max_seq_length, tokenizer)\n",
    "logger.info(\"***** Loading training set *****\")\n",
    "logger.info(\"  Num examples = %d\", len(train_examples))\n",
    "train_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)\n",
    "train_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)\n",
    "train_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)\n",
    "train_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)\n",
    "train_data = TensorDataset(train_input_ids, train_input_mask, train_segment_ids, train_label_ids)\n",
    "train_sampler = SequentialSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=train_batch_size)\n",
    "\n",
    "\n",
    "train_labeled_features = convert_examples_to_features(train_labeled_examples, label_list, max_seq_length, tokenizer)\n",
    "logger.info(\"***** Running training(labeled) *****\")\n",
    "logger.info(\"  Num examples = %d\", len(train_labeled_features))\n",
    "logger.info(\"  Batch size = %d\", train_batch_size)\n",
    "logger.info(\"  Num steps = %d\", num_train_optimization_steps)\n",
    "train_labeled_input_ids = torch.tensor([f.input_ids for f in train_labeled_features], dtype=torch.long)\n",
    "train_labeled_input_mask = torch.tensor([f.input_mask for f in train_labeled_features], dtype=torch.long)\n",
    "train_labeled_segment_ids = torch.tensor([f.segment_ids for f in train_labeled_features], dtype=torch.long)\n",
    "train_labeled_label_ids = torch.tensor([f.label_id for f in train_labeled_features], dtype=torch.long)\n",
    "\n",
    "train_labeled_data = TensorDataset(train_labeled_input_ids, train_labeled_input_mask, train_labeled_segment_ids, train_labeled_label_ids)\n",
    "train_labeled_sampler = RandomSampler(train_labeled_data)\n",
    "train_labeled_dataloader = DataLoader(train_labeled_data, sampler=train_labeled_sampler, batch_size=train_batch_size)\n",
    "\n",
    "\n",
    "train_unlabeled_features = convert_examples_to_features(train_unlabeled_examples, label_list, max_seq_length, tokenizer)\n",
    "logger.info(\"***** Running training(unlabeled) *****\")\n",
    "logger.info(\"  Num examples = %d\", len(train_unlabeled_features))\n",
    "logger.info(\"  Batch size = %d\", train_batch_size)\n",
    "logger.info(\"  Num steps = %d\", num_train_optimization_steps)\n",
    "train_unlabeled_input_ids = torch.tensor([f.input_ids for f in train_unlabeled_features], dtype=torch.long)\n",
    "train_unlabeled_input_mask = torch.tensor([f.input_mask for f in train_unlabeled_features], dtype=torch.long)\n",
    "train_unlabeled_segment_ids = torch.tensor([f.segment_ids for f in train_unlabeled_features], dtype=torch.long)\n",
    "train_unlabeled_label_ids = torch.tensor([-1 for f in train_unlabeled_features], dtype=torch.long)\n",
    "train_semi_input_ids = torch.cat([train_labeled_input_ids, train_unlabeled_input_ids])\n",
    "train_semi_input_mask = torch.cat([train_labeled_input_mask, train_unlabeled_input_mask])\n",
    "train_semi_segment_ids = torch.cat([train_labeled_segment_ids, train_unlabeled_segment_ids])\n",
    "train_semi_label_ids = torch.cat([train_labeled_label_ids, train_unlabeled_label_ids])\n",
    "\n",
    "train_semi_data = TensorDataset(train_semi_input_ids, train_semi_input_mask, train_semi_segment_ids, train_semi_label_ids)\n",
    "train_semi_sampler = RandomSampler(train_semi_data)\n",
    "train_semi_dataloader = DataLoader(train_semi_data, sampler=train_semi_sampler, batch_size=train_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 0/46 [00:00<?, ?it/s]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f430f6fdccc14ed796661e16d12adec8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "188204d740394e28ac078126ea08cbd8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fafc2dc3866d44dd9044da4321fbbd27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 1/46 [00:36<27:14, 36.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[24  4  7 17  4  0 44]\n",
      " [ 0 99  0  0  0  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 7  2 17 74  0  0  0]\n",
      " [12 20 13  5 40  3  7]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 5  4 21  5  1  2 62]]\n",
      "0: loss=0.583169705592669, (u, l) = (0.95,0.455)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35f6c032f5e74dd1b43e205b6a540b5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b3b56404adb448b8e3e97bac536b350",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af1473ffa2834a80af0947f8b69e5ca0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 2/46 [01:25<29:22, 40.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[26  5 10  5  4  2 48]\n",
      " [ 0 99  0  0  0  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 1  0 11 87  0  1  0]\n",
      " [15 10 24  1 43  4  3]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 4  1 26  0  0  2 67]]\n",
      "1: loss=0.5705461066502792, (u, l) = (0.9401,0.456)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9440cfe5edd2451fbe9f25a881cf04f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fbbf04f1bd64172baf784b13cf4ec0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5a9c25ec8d24770ae0617f4fe166816",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 3/46 [02:15<30:55, 43.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[35  1  3  4  5  3 49]\n",
      " [ 0 99  0  0  0  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 1  0  0 95  3  1  0]\n",
      " [16  7 34  3 35  4  1]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 1  0  0  0 17  2 80]]\n",
      "2: loss=0.5447999238967896, (u, l) = (0.9302,0.457)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a450b4c552d44089a6c749fddbb636d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9b6a040d2534f03bc47329e8b44e2d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "581a8d2d1e224cf7836e56f297b727bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   9%|▊         | 4/46 [03:06<31:53, 45.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 30   2   2   7   3   6  50]\n",
      " [  0 100   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0  98   0   2   0]\n",
      " [ 11   5  34   1  45   4   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   3   0   0   1  96]]\n",
      "3: loss=0.5177683778680288, (u, l) = (0.9203,0.458)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49fb83aac207482b9a4e98335367df1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f3d169c3df54b9cbab1f99ed29c856d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0596c29ee427434eae56e3198b3649e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  11%|█         | 5/46 [03:58<32:21, 47.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[36  1  0 13  0  6 44]\n",
      " [ 0 99  0  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 98  0  2  0]\n",
      " [15  3 30  1 49  2  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 2  0  0  0  1  0 97]]\n",
      "4: loss=0.494927616073535, (u, l) = (0.9104,0.459)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2aebdb16d94d4495857eabad335641cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "765dfdb275744ecf869789ec94d9876d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f6408e4978d4a89b7b6210fb8c2a127",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  13%|█▎        | 6/46 [04:50<32:29, 48.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[25  1  0 21  0 11 42]\n",
      " [ 0 99  0  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 8  4 28  1 56  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 1  0  0  0  0  0 99]]\n",
      "5: loss=0.4764009095155276, (u, l) = (0.9005,0.46)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cfeac9708f14dd5a591bd76244a6eb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3e21deb6e8e448cae9bae9788855042",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5485ddfbcd344c038a32de16a5de539b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  15%|█▌        | 7/46 [05:43<32:30, 50.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[37  1  0 15  0  4 43]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 6  3 33  1 55  2  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 1  0  0  0  0  0 99]]\n",
      "6: loss=0.46056690124365, (u, l) = (0.8906,0.4609)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51d6a53784294859a0de932470bdbc2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6c9ea77a14b4ad6b4978d08e7d3f11a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c00b44f6d6c44509ba6e4dc33eb0921",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  17%|█▋        | 8/46 [06:36<32:13, 50.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[43  1  0 12  0  4 40]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [11  2 24  1 60  2  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 1  0  0  0  0  0 99]]\n",
      "7: loss=0.447547598527028, (u, l) = (0.8807,0.4619)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6c4b86aae7c49d7a02406de3eb9f3d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c6c6eee4307450ebe4010e27a5c2743",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97f898289ef148fea0a6dcee90b7ab45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  20%|█▉        | 9/46 [07:30<31:57, 51.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 35   1   0  11   0   3  50]\n",
      " [  0  98   1   0   1   0   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0  99   0   1   0]\n",
      " [ 10   2  24   1  61   2   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0 100]]\n",
      "8: loss=0.4358903616666794, (u, l) = (0.8708,0.4629)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a666359c499427dbbc3e0d60a01adfa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7c916cc358247a8858f2e00f405f129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ea5ccfbbc604f59996bdb0ec58fef0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  22%|██▏       | 10/46 [08:23<31:24, 52.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[44  1  0 14  0  4 37]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 8  2 23  1 64  2  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "9: loss=0.4218651858659891, (u, l) = (0.8609,0.4639)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2fe7274977f4fc4beff3275105508b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f234de6dcd2408da63c2f99ae0680a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39a0cc7a9ad54f04b0acc9d4440ba9ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  24%|██▍       | 11/46 [09:16<30:42, 52.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[39  0  0 11  0  8 42]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 7  2 19  1 69  2  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "10: loss=0.4089169674194776, (u, l) = (0.851,0.4649)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12c580d87b894bfdbe8282ab22fd3fad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "522aac8a81784968898ed6f4e11c2838",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a22711de234f4d4f966dc7f8ef388346",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  26%|██▌       | 12/46 [10:10<29:57, 52.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[42  0  0 10  0  8 40]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [11  2 16  1 68  2  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "11: loss=0.3965758841771346, (u, l) = (0.8411,0.4659)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "505598a2b1704542ac61511ac3a58c99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c622889739e467fbe15b8dc46c129aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcd593343b1e4a24b6c8c3f214252cd9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  28%|██▊       | 13/46 [11:04<29:15, 53.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 40   0   0   8   0  11  41]\n",
      " [  0  98   0   0   1   1   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0  99   0   1   0]\n",
      " [  9   2  18   1  68   2   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0 100]]\n",
      "12: loss=0.385424493597104, (u, l) = (0.8312,0.4669)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f72cfea1478343ffba699f2595a8d05e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc14e9df944b4ce59464091dd4fca1ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a50d68eba8ce46e8a3d20e44bbf947e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  30%|███       | 14/46 [11:56<28:14, 52.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 39   0   0  10   0  11  40]\n",
      " [  0  98   0   0   1   1   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0  99   0   1   0]\n",
      " [ 10   1  18   1  68   2   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0 100]]\n",
      "13: loss=0.37321917368815494, (u, l) = (0.8213,0.4679)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd47a2f4fc7c40f8a94392bd3614b116",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1856de65e2e34ff789e20c731b53db99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c890b0b861e74dd3a7006f2de668fe51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  33%|███▎      | 15/46 [12:50<27:30, 53.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 39   0   0  10   0  14  37]\n",
      " [  0  98   0   0   1   1   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0  99   0   1   0]\n",
      " [  9   1  17   1  69   3   0]\n",
      " [  0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0 100]]\n",
      "14: loss=0.3616527651364987, (u, l) = (0.8114,0.4689)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08843f15ed034161b7dd5c687f363be4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b875a2323124a4ea362f6fa93112758",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02113e9d6d4f46d6a46d57e372ecc472",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  35%|███▍      | 16/46 [13:44<26:41, 53.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[47  0  0  7  0  9 37]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 9  1 16  1 70  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "15: loss=0.3502768518833014, (u, l) = (0.8015,0.4698)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63a59500e3ee4305a5523a87d569fcc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef566f9375324c35a2b2e1e61dfea26c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed1fd32fd2b54c1f8e42ae89fb5a884d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  37%|███▋      | 17/46 [14:38<25:52, 53.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[44  0  0  6  0 10 40]\n",
      " [ 0 99  0  0  0  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 9  1 15  1 71  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "16: loss=0.3386179839189236, (u, l) = (0.7916,0.4708)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e20a98c19a24490a320f3e884e3a66f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb9553a6a61d4b00a09abc1cf0f359e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19120799abca475c9e2da931015f64b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  39%|███▉      | 18/46 [15:31<24:55, 53.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[41  0  0  9  0 12 38]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 8  1 14  1 73  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "17: loss=0.3270162229354565, (u, l) = (0.7817,0.4718)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c1748921542423dbbb70aa3d990f6a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24b2cda483bc45be86309e79c4270ad9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "274c2d9f25614a229ea219bf06e5528a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  41%|████▏     | 19/46 [16:25<24:05, 53.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[48  0  0  8  0 11 33]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 9  1 15  1 71  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "18: loss=0.3154724801962192, (u, l) = (0.7718,0.4728)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42be5dc5befd40ffbef964c8a36e4981",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92fe993aaae243c9ac7aa90847b27aad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "260d64782a79456b9ef91cf9d2304ce4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  43%|████▎     | 20/46 [17:18<23:13, 53.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[48  1  0  5  0 11 35]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [10  1 15  1 70  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "19: loss=0.3043116835447458, (u, l) = (0.7619,0.4738)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31c98c633fe34b03ba9a1a79fe4acd3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1cce8afc5ad4e11bbcb7025cd735cfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4529a18ff6644618d0d740d7bf3ce81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  46%|████▌     | 21/46 [18:11<22:15, 53.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[40  1  0  8  0 13 38]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 9  0 15  1 72  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "20: loss=0.29329631878779483, (u, l) = (0.752,0.4748)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9eeb8180f8cf4639a1c8859c8a67da52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "649e7b0eb9c6493d8b7ea534b51223f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e60625692584209925ce56d1d3f8d86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  48%|████▊     | 22/46 [19:05<21:22, 53.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[34  1 17  8  0  0 40]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 8  0  3  1 73 15  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "21: loss=0.2816082422549908, (u, l) = (0.7421,0.4758)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a27c0c8664714a8fa5bc6c027ac59b2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17e84dd15a134584a829ddd011910c7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3526782ad1f4221808bdfed635e04d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  50%|█████     | 23/46 [19:59<20:31, 53.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[37  1  0  6  0 14 42]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 8  0 15  1 73  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "22: loss=0.2706518998512855, (u, l) = (0.7322,0.4768)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ed72e32bd1040f390891ef87761d365",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "330850bf2131497abb1a65c01bc1f0f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "265c53cc2838437186f6bc14548aa4a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  52%|█████▏    | 24/46 [20:53<19:40, 53.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[39  1  0  6  0 13 41]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 8  0 15  1 73  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "23: loss=0.259867970760052, (u, l) = (0.7223,0.4778)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c87278a70d0c4718a91567eaf50bfa15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "adef4953b0114b14bcf4f53f641e0e29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "314b7f80f8184eaab4c545894a26931c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  54%|█████▍    | 25/46 [21:46<18:45, 53.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[46  1  0  9  0 10 34]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 7  0 14  1 75  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "24: loss=0.24830834911419794, (u, l) = (0.7124,0.4788)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e91d0e18fa204c2f92c8443f8d3a1bce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2365beeeb2414efca74360c67d52cc2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2cc82ab26b494702bb4033448c77c0a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  57%|█████▋    | 26/46 [22:39<17:48, 53.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[42  1 18  7  0  0 32]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 7  0  3  1 75 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "25: loss=0.2370733882372196, (u, l) = (0.7025,0.4798)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0640376d79354bc4a603d703a25b6be4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "389f8263fa3c4e4bb3ba4333a6b075b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9538398d0fe545e69412cce84d02a36f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  59%|█████▊    | 27/46 [23:33<16:57, 53.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[46  1  0  9  0 12 32]\n",
      " [ 0 98  0  0  1  1  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0 99  0  1  0]\n",
      " [ 8  0 14  1 74  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "26: loss=0.22592863555137926, (u, l) = (0.6926,0.4807)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d7a052225e04689a2f93c01f480066a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3a3615f058e4dc7a6d913f3889a0895",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "561f7119429843c28bd98259bb014d1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  61%|██████    | 28/46 [24:26<15:59, 53.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[44  0 16  6  0  0 34]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 8  0  3  1 74 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "27: loss=0.21512280519192034, (u, l) = (0.6827,0.4817)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89e1a0555a264914971d9fe4944ffc6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d847589bfc34cbe8299f0b8b776f24c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcaad7c1ba2744d0bb9586063226b579",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  63%|██████▎   | 29/46 [25:20<15:10, 53.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[38  1 15  7  0  0 39]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 8  0  4  1 74 13  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "28: loss=0.20412253187252924, (u, l) = (0.6728,0.4827)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47546f09ade84a0c942ae0343f0381a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f99fc12a1134f28b85ef5ec51c592b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "313cf138d50144f4afd17f5093da47b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  65%|██████▌   | 30/46 [26:13<14:16, 53.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[39  1 17  6  0  0 37]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 7  0  5  1 74 13  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "29: loss=0.1928740808596978, (u, l) = (0.6629,0.4837)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6247e815e914c1da79bd3d9ab1ad678",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ada402a249004be184b7f2c8d96edafc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8da4e4c391774bf5b434d63dd799bc57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  67%|██████▋   | 31/46 [27:06<13:19, 53.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[42  0 17  6  0  0 35]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 73 13  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "30: loss=0.18190892384602472, (u, l) = (0.653,0.4847)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8f9f10f3be14daab66b971f76808689",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23e8f44c874c4040b96c5658ea3644cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8702d105c22d48c0a4fa9aa44dbcdc2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  70%|██████▉   | 32/46 [27:59<12:26, 53.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[39  0 16  4  0  0 41]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 5  0  5  1 75 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "31: loss=0.17091390261283287, (u, l) = (0.6431,0.4857)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca7ea7a80ff647eeafc4198a645aec39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2901ed3555604d999c1810ef0b44ade7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd10dc0077d14a1f911b4a2e373a19d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  72%|███████▏  | 33/46 [28:53<11:34, 53.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[45  0 17  5  0  0 33]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 73 13  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "32: loss=0.16048299005398384, (u, l) = (0.6332,0.4867)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3827723749954be8a6da033ba06156bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c5681ee85494dac993ea51a6a59bc12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1d3989eaf9d4e298474865acad33870",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  74%|███████▍  | 34/46 [29:47<10:44, 53.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[40  0 18  6  0  0 36]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 73 13  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "33: loss=0.14960603415966034, (u, l) = (0.6233,0.4877)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e08dd466e034dd9974473f2a1cc7d8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0be58fca6430446f9b19f18beed03598",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f134d7e99dd04fc9907235378d18c105",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  76%|███████▌  | 35/46 [30:41<09:50, 53.67s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[40  0 20  6  0  0 34]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 74 12  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "34: loss=0.1382812846165437, (u, l) = (0.6134,0.4887)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "084621f3d6cd4fb0b0c0ff18ac10cdb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31d7192f1dc34b12918fc25a1f2b03b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c51143ef15e9411cb08287b19c1bbefd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  78%|███████▊  | 36/46 [31:35<08:57, 53.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[39  0 20  6  0  0 35]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 7  0  5  1 73 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "35: loss=0.12749346632223862, (u, l) = (0.6035,0.4897)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64a4a7f79664468aa09de7519d315635",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2481c34c15ca460b926e72f672e96f3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55523c6d2a57433db1e4362ef6005c24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  80%|████████  | 37/46 [32:29<08:03, 53.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[43  0 18  6  0  0 33]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 8  0  4  1 75 12  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "36: loss=0.11622250080108643, (u, l) = (0.5936,0.4906)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "076c9845640f445288f9bbd7a202735b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "986ba0ab753040e6aac4c27eccf8a9be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3fca265e7c5406291215e7c2b613b65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  83%|████████▎ | 38/46 [33:22<07:08, 53.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[43  0 18  6  0  0 33]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 72 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "37: loss=0.10577617127161759, (u, l) = (0.5837,0.4916)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47ae23003c954c979a2c8f3d36cdf170",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97814c2303a744b0b2a789c788eacd40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40fe8a19323e46e2bd5e7fd4965e9b1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  85%|████████▍ | 39/46 [34:16<06:16, 53.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[45  0 18  6  0  0 31]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 72 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "38: loss=0.09502907326588264, (u, l) = (0.5738,0.4926)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2f7639e1792426486794e0820374e1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4b12b3e7b3b4dcb9ddbeb9ed9b98fae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24dc0cae76d4408fb6dc16904273829a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  87%|████████▋ | 40/46 [35:09<05:21, 53.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[44  0 19  6  0  0 31]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 72 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "39: loss=0.0841589283484679, (u, l) = (0.5639,0.4936)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "155bbd265ac949ebafbad009eeef5dd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3ed9043002e419dbad1272f6d3d3b4c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a9ed01826b742bebd78ec7dcff95086",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  89%|████████▉ | 41/46 [36:03<04:28, 53.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[44  0 18  6  0  0 32]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 72 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "40: loss=0.07356591751942268, (u, l) = (0.554,0.4946)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a742a1a5455e4c868f24231f22ebf02a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4132e63811de4ced9778b776b8472078",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abd1695365fd444887144a24550879f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  91%|█████████▏| 42/46 [36:57<03:35, 53.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[46  0 18  4  0  0 32]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 72 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "41: loss=0.0627681899529237, (u, l) = (0.5441,0.4956)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a24b5ee1119541b48562d3b3d21a6b49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec61cbfba6634367819e9517c41707c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81fe166c1fde454b886570e587a98de7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  93%|█████████▎| 43/46 [37:51<02:41, 53.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[47  0 18  4  0  0 31]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 72 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "42: loss=0.05222095663730915, (u, l) = (0.5342,0.4966)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2dce0d1c17ae4898af5796336d85b900",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ac25ed2c4314c37a5a528f9b6ccf1e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1cfdc9d97c140efb195ab6d858b104b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  96%|█████████▌| 44/46 [38:45<01:47, 53.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[48  0 18  4  0  0 30]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [ 9  0  4  1 72 14  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "43: loss=0.04116907715797424, (u, l) = (0.5243,0.4976)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42ca72dc41dc49de9bd706dcd1ca202a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb2952264e3b4641906f17ea33c8aa6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01d7d8559f8c45a8b52a59e9a71fba2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  98%|█████████▊| 45/46 [39:39<00:53, 53.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[49  0 17  4  0  0 30]\n",
      " [ 0 98  1  0  1  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [10  0  4  1 74 11  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "44: loss=0.030981563604795016, (u, l) = (0.5144,0.4986)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bc2ccc5c7cd4b518f7cf347540faab6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (labeled)', max=4, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "538cf728e18c42b1b8c2ba00ea711b7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration (all train)', max=52, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7434525602f84906815ba95e7e0fb0d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=2, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[74  4 16  2  0  0  4]\n",
      " [ 0 99  1  0  0  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  1 99  0  0  0]\n",
      " [24  1  4  1 67  3  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  0  1  0 99]]\n",
      "45: loss=0.02850024516765888, (u, l) = (0.5045,0.4996)\n",
      "{'ACC': 87.6, 'ARI': 79.27, 'NMI': 80.42, 'u': 0.5044999999999996, 'l': 0.49955000000000005} ['SearchCreativeWork' 'BookRestaurant']\n",
      "Confusion matrix, without normalization\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAIWCAYAAABjtEbrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XecFPX9x/HXG5ADBKx3alRAzc8ONixExd67RmOJxhijRqMxNmwoVuwtlkgSJZqoMVFjQWPsBRBEBewVBBU8UKRIU/j8/pg5XC4ct3fsMrvL++ljH+zOzn7nPXN77vc+3+/MKiIwMzMzKwUtsg5gZmZmVscdEzMzMysZ7piYmZlZyXDHxMzMzEqGOyZmZmZWMtwxMTMzs5LhjomZmZmVDHdMzMzMrGS4Y2JmZmYlo1XWAczMzGzRtOzYOeL7GUVrP2ZMeDIidi/aBnK4Y2JmZlbm4vsZVK1zSNHanzn8lhWL1ng9HsoxMzOzkuGKiZmZWdkTqDJqDZWxF2ZmZlYRXDExMzMrdwKkrFMUhCsmZmZmVjJcMTEzM6sEFTLHxB0TMzOzSuChHDMzM7PCcsXEzMys7Pl0YTMzM7OCc8XEzMysEniOiZmZmVlhuWJiZmZW7oTnmJiZmZkVmismZmZmZU+eY2JmZmZWaK6YmJmZVQLPMTEzMzMrLFdMzMzMKoHnmJiZmZkVlismZmZmZa9yvivHHRMzM7NyJzyUY2ZmZlZorpiYmZlVggoZyqmMvTAzM7OK4IqJmZlZ2aucya+VsRdmZmZWEVwxMTMzqwQtfFaOmZmZWUG5YmJmZlbuhOeYmJmZmRWaKyZmZmaVwFd+NTMzMyssV0zMzMzKnq9jYmZmZlZwrpiYmZlVggqZY+KOiZmZWSXwUI6ZmZlZYbliYmZmVu6kihnKccXEzMzMSoYrJmZmZpXAc0zMzMzMCssVEzMzs0rgOSZmZmZmheWKiZmZWdnzJenNzMzMCs4VEzMzs0rgOSZmZmZmheWKiZmZWbkTnmNiZmZmVmiumJiZmZW9yjkrxx0TMzOzSuDJr2ZmZmaF5YqJmZlZJaiQoZzK2AszMzOrCK6YmJmZVQLPMTGzXJK2k/SUpOclvZTe/4Wk1gVou4WkuyQNkTRS0hbNbGcbSf9d1DylQtIfJR2VdY5KIGk9Sc9JelnSG5J65Tx3tKTh6Xu77vaVpAMX0t6xkgam7Y2UtN3i2RMrd66YmBWApN2B24GdIuKjdNkewCPACGD4Im5iB2CriFhb0lbA7Ga2MxA4eBGzFJWkLsCoiMjnz78zgJlFDbTk+CvwVEScJ2kF4ENJIyLiPyTvt99GxMsAkpYCPgCeXFBDkg4GdgJ6RsQcSb8EVl4se7Gkkk8XNrOUpBbArcBldZ0SgIh4QtK/CrSZ1YAv0nZfaW4jERHA5AJlylxETMs6QwXZALgYICK+kvQhsAnwn4i4p966uwEvR8S3DbR1AXBIRMxJ27uzSJmtAlVG98osW5sCawALGiI5BngTQNJKkh6Q9KKkVyT9Il3eKX0c6dDP05Lel/ST9Pl9gXOAjdMS+s/q1k+fXyMts4+u26ik/SQNlvRMOqTUQ1J17uvS9TpI+ktabh8sqZcSrdNthaSTJD0u6WNJByzoAEjaV9J7kl6QdJWk1yQ9me7b3elzl+Wsv4GkAWm2wZKOS5cvA9yX3q8bMuicc3yOlvRfSbMk9ZY0WlL/dP2X03X+nm73A0lvSera2A9Q0gHpsXo6/flsnfNcK0lXSBqUPne/pDXT52ok/TNdPljS9ZLaKhl2mylp+3S9h9NsXdLHD6bPnynpEUkT0n3bNj1uT6fb279eztPTY/Fc+jPZVNKf07YHS2ovaRdJn0ga1th+1zMA2CfdzpokHZUhDax7BHB3A8eyBliP5P36nJJhzeObmMWaQyrebTFyxcRs0a2V/vt5/SciYkbOw7+T/JXZR9KKwJuSPomIlyQdCowCZkTEzpLOAi4EdouIRyQtDxwdEdsDSBqSrk9EjJJ0KtA/Z1t/ArpGxJeS9kvbGZyznTo3AC0jYhtJbUk+iL6IiLuB7dNOzNIRsaekQ4ArgIcWsJ91GW8GjgR6Aa8DtwEHAMsCYyXdHBHjgPbAxRExRMmwwEhJz0XEh3UZ6/Y13d+63HMjYldJpwH3Ai2BLmmGbSQ9DtSSVJdGAr9YyF/1uZYGDo6Ir9POw4tAp/S5s4DNgG3TYYmbgZ7AJ8DfgIERcbCSuUSvACtFxFGSeuYcn/1yO4QRcaCSjuR6EbFv2oFZGugAnBQRH0nqCLybHpfJkg4HfglsERHTJZ0B7BsRxyqZc/SHtIL0lKShwLHpsbsB2LiB/R4eEaem938FPCLpY2B54LSIeLb+CyS1BzYHft5Am11IvrnlAGBnoAYYKmlyRNzXwGvM5nHHxGwxkLQqyZj7MQARMVHSYyQfNC/lrPqf9N+RpB8szfQ18Ov0Q/RRFjAXQMkQ1BHArmmmGZL+kWbK/Ws4N9MajWz3/YgYk7b/NvBlRMwGaiVNSF8/DvgQuELS9STzF1YhGTb4sJH2H06zXpduo/7zx5NUqH4E3JpnpwSSeUB3Kplb8T2wuqSaiKglOR6X1A1LAJcDVenPdBfSn1NEzJZ0DDAhz23m7s/z6f50Aq5M//0OWAFYBxia5rg/Iqanr/0TsHp6/y7gKOAeSdXAzLphrpyOR2MeBAZHxA6SVifp4AyLiNfrrXcg8O+c41FfFUk1/g/pOuMk3U3y3nfHpIgW8PtQljyUY7bo6uaVrLqQdVZL/8390JqQsxyAiJiS3p0JLMrZPLuked4D/kHywV9fNcmHyEIzAbmZlmpku1Nz7n+/gMd1+3QdyV/S26aVkeFAu0baJiIWOj8mIsYC1wLbkEz0zdcjwEsRsU1OpaYuz2rkHKOI+CIiRrGAn2lEDG9CZwj+d77PXcDEiNg6zTF+ITkmR8Rb6cO/AztIWgU4nCZ2ACStR1LduCFteyzwFHDmAlY/gqRS1JBJ6b9f5iz7jP99X5ktkDsmZovuDZK/9Het/4SS+RubAmPTRdU5T1eT/A+7OWan7Velj5et9/z3EfEbkgpFLfMP89SZAMwqYKam2AJ4Ouev7sY6PHmR1IZk2OJdoE+er6khGX74T/q4fpax5BwjSSukwz3/8zOVtKak5dKHs0k6fkiq//NpyBbMX93KzVI/x9KS1gFIh8eeI+k07ELSqahb7wbNf5pv7u2GdLW6DuN3Odv7DuiYG07SSiRDVQs7y+xDYDpJx7NONenkbSsOkVRMinVbnNwxMVtEETEXOA44R9KP65YrOUWyK8k4/hckHxZHp8+tAOwFNPdshVqS//lvmD7eo97zj0lqmc5xGUoyF2NBue8C6ibhtgUOWYRMTfERsGW63VWAbjnPTU2Xt5N0tpLTo/N1Ack8mF8Dv5HUPY/XfAV8U5cH2L3e8/2BIyXVHcMrgI0W8DOtAu7nhw/5Ufzw89kzz/y5x6Ub81e6+gOHSKqroJxaL+tdwOnAB7nDLBFxakRs38CtbpjnPZI5Uoel2+4A7AvUn2NyKFD/DB0kHSHpunR7s9Isv0qfWxr4GcnpyGaNcsfErADSOQJHAf2UnKHxMslwwp5pBwCSyYLdJL1IcgbEOenE1+X54UyUh9P5BTcAKys5u2Nf4Gx+OCunOiK+J5lg+g9JDwMfp+v/M93WC8BLkp4nmQNxUjr3IPeMl5bAaUCkeZ8j+dD5W7pO3VlG92n+s2X+5+wjSTvmZLxIyVk2uwNHKznj5VaS61jcIGl9kgml60kaDFxG8iF+tqQdI+KrNMeLJPNyxtbLvX56/0iSTsHuks6T9HfgBJKzSdYh+Yv/EUlHp+t/kVav6v/s5pDM3zhP0pP80EG5L/3ZXA28BrycHqfaiHg452faNf2ZPgtcERF1QxiXAMdJepofKh/3SVpV0l05x+PwnDi/AX4q6bn05/Zl3TFLT9m9E3hW0gskVZ5bcl77EMkE2gWeLbMwaWdif+BQSS+RHPuHgBvrrXoYC+iYkHSg1sp5fAbQStJrJMelPwsf/rFFpSLfFiMllzUwM6tckn5EMo9l9fRDuOIomcz8ZETsknUWW/xaLr9GtN35wqK1/+0/f/laRORTgVxkPivHzJYEV5FcubTiOiWStiSZcLoWOXNLzMqVOyZmtiQ4qbEzesrYKiRn5XwG7JdxFstQpZwu7I6JmVW8Cu6UEBH/Bv6ddQ6zQnHHxMzMrAJUSsXEZ+WYmZlZyXDFxBZJy7bLRKuONY2vWKLWX3WZrCM023dz5ja+Uglr3aq8/y6qjL9Ny9PcMj+ZdPgbr02MiOrG12yaSqmYuGNii6RVxxpWPaL+pQ7Kx3OX7pZ1hGYbP3lm1hEWyWrLt806wiJp1bK8O1blbObshr6mpzwst3SrT7POUMrcMTEzMyt3GVwIrVjc5TczM7OS4YqJmZlZmROL/8v2isUVEzMzMysZrpiYmZlVAFdMzMzMzArMFRMzM7MKUCkVE3dMzMzMKkCldEw8lGNmZmYlwxUTMzOzcucLrJmZmZkVnismZmZmFcBzTMzMzMwKzBUTMzOzMlcKl6SXdCbQBZgI/B/wK6AtcAXwSbrs3Ij4cmHtuGNiZmZmi0TSysA5wIoRMVfSw8CBwLbA0xFxv6R9gGuAIxfWljsmVjK+m/wl4x84l1YdqgGYO3s6rVfsQvVupwHwzZD7mPz6w3T+zb1ZxszbjBkz2Hm7n7DjTjtzSd+rs47TqAm1X3LjlRfz/jtv8s8nXgQgIvjbX24D4PPPxjB18mQuu/62LGM26pOPP+aSPr3ZaJNN+eLzz1h++RU4+7zeWcfK27PPPM3DDz1IdU0Nkjiv94VZR2qScs9fbr+3uTKumEwHZgMdgW+A9sDbJNWSy9J1BgJ/bawhd0ysZLRo3ZYVd/otbTtvAsCkwX+nbaeNAZgxdiRzZk7LMl6TXXZRb7pttHHWMfL2+tBB7LTbXrz39sh5yx554D46LLMs+x98OADvv/NWVvHyNmnS1xx0yM/Ye5/9AOi+8YbstseebLLpZhkna9z06dM5+aQTeH3E21RVVXHoIQfx3LPPsMOOO2UdLS/lnh/K7/e2VETElHQo5x+SxgGfAR8BNcDUdLUpwHKSWkXE9w215cmvVjJatu04r1MS33/HrC8/pM2qGzDn20l8+8FLdNx4n4wT5u++e/7Glj22pnOXLllHydtuex/A0u3bz7fssQf/weRvJnH3n2/l+r59aLf00hmly99m3Tef1ykBmDt3blnkBhjyymA6depMVVUVAD16bM0Tjw/IOFX+yj1/Of7ezkdFvMGKkobl3I6bb9PSxsCZwF4RcTTJPJMLgFqgQ7paR2DSwjol4I5J0UjaQtLzkgZJ6iPpFkl/kHSVpOeb2eYFkkLSCfWWryhpqqR7JP24Ge3eKWmT5mQqlmnvP0/7dXoSMZevB/6V5bY+KutIeXvv3Xf44P132We/A7KOssi++GwM06ZO4chjT2T/Q47g10ccwJw5c7KOlbdHHn6InXfZlXXWWTfrKHmpra2lfYcO8x537NiRCbW1GSZqmnLOX0m/t0UyMSK659z61Xt+VeDrnE7HOKANMADokS7bOn28UO6YFElEDAWeBwZFRJ+IOAnoBoxehDYvBl4DzpFUlfPUScAY4J6I+KgZTR8TEW80N1cxfPvhyyy9dk9mf/kxatGKqSOfYOrIAcT3s/hm6P18N+nzrCM26LFH/02bNm24/poreWXQQF577VVuu/nGrGM1S/sOHdlo080BWGOt/+PbqVMY/8VnGafKz4vPP8dLLzzPFVdfl3WUvNXU1DBt6tR5j6dMmUJ1TU2GiZqmnPOX/e+tkjkmxbrl4T/A25KuldQb2BzoC5wL7CLpfJLJsGc01pDnmCwmkloBKwLDgEPSZWsC1wGDgK7AtSTjcvcAVSQzl1cDbgLOiIjngbeAdsCxwC2SaoAAJuRs6wpgq4jYXtIhQL+IWFZSW+Bm4D1gdeBF4APgJkn9I6K/pPVIynHvknSk/hERjxXtwCzAjLEjqVplPdSyFVUr/x9VK/8fkEyOnfrWUyy7xSGLM06TnXHWufPuz5o1k2+nTeM3v/1dhomab6tttmfsp6MAmDZ1CnPmzGHF6pUyTtW4/zwxgEEDX+aqa29g/LhxjBnzKVtu1aPxF2Zsy616MGbMp8yaNYuqqioGDx7I8SecmHWsvJVz/kr6vc1CRMwh+SN5QX7dlLZcMSm+n0jqA1xPMjM5t645G7g4Iq5Knz8vIiYCJwLLR8QY4HPggbRTUucioJekNiRvhFvrbfOPdXci4n6SGdIA6wAbA38h6bWOjoiRJJWdOncAf4yIq0k6KNG83W6+qW8+Qcdue8y37LtvvmDKiEeJ72cxach9zP1u5uKO1WSP/PtBBr38Eq++OoR/3X9f1nEaNXTwSzz8r/uYUDue2264kpkzZnDsSb/nvbdHcvtNV9P3wrPpe2M/qtq0yTrqQr3x+msc/fPDeHXIEPbcdScOPfgAPvzg/axj5aVdu3bcdPNtnH7qKfS54Hy6du1WVhNHyz0/lN/vba6MKyaF24+Ixf65s8RIOyTtI+KMnGVdgP5pNWMl4HRgMsmkoC0jYvt0vUeBG4HtgJsiYkK6vH9EHC3pPmAUMCMiLk7nrVwTEY/lbiN9zeiI6JLePxY4nKRTdG5EvJ7mHJ1WTL4FVo+IrxeyX8cBxwG07FC9Wadj+y/agcrQ65fulnWEZhs/ufQ7Zwuz2vJts46wSFq19N91WZk5u3zmOS3Icku3ei0iuheyzaWq14oV9r+ykE3O58s/H1zwzA3xb1a2zgamRcRlJFWMXDcCvYB2dZ2Sei4i6RzcsoDnppJ0dEirKtXp/TWAVyJiR+APQJ8FvHYEsFa6/qrpBXHmExH96iZAtWy7TKM7aWZmxVcpFRPPMSkSSd2BnkBrSQdFxAPpUycCnSXtATwA9E0nsrZOl+8UEc9ExNOSbiBnopCk04BudesAK6TLDwI6Az+TNCIixkoaLulykssAT07P5HkOOF/SG0An4HZJG6Y5u0p6iuQSwmdJep9kfkvfoh4oMzNbZKVwSfpCccekSCJiGLDjApafBZyVs2jbnPtnAqQdldnAixExIue115FMlq3f5gMknZzcZcfkPPxzzv1DFxA3N+fnwC8XsI6ZmVnRuWNSmu4kuTjNP7IOYmZmZaIyCibumJSiiDg86wxmZmZZcMfEzMys3CnzL/ErGJ+VY2ZmZiXDFRMzM7MK4IqJmZmZWYG5YmJmZlYBXDExMzMzKzBXTMzMzCpBZRRMXDExMzOz0uGKiZmZWQXwHBMzMzOzAnPFxMzMrMxJ/nZhMzMzKyGV0jHxUI6ZmZmVDFdMzMzMKoArJmZmZmYF5oqJmZlZJaiMgokrJmZmZlY6XDExMzOrAJUyx8QdE1skG662DAOv2jPrGM223Oa/zTpCs3015A9ZR1gkLVpUxv9EbfFr07pl1hGsiNwxMTMzK3eqnIqJ55iYmZlZyXDFxMzMrMwJqJCCiSsmZmZmVjpcMTEzMyt7lfMlfq6YmJmZWclwxcTMzKwCVEjBxB0TMzOzSuChHDMzM7MCc8XEzMys3KlyhnJcMTEzM7OS4YqJmZlZmROV8/1TrpiYmZlZyXDFxMzMrAJ4jomZmZlZgbliYmZmVgEq5Tom7phYyXr2mad5+KEHqa6pQRLn9b4w60gLdfIRO/CjmmX5dsYsqlq3ovdNj7DROqtx0uHb894n41lvrVW4+NbHGDt+UtZRGzV+/Hgu7nM+b44cyUuDhmYdp8nK7b2Tq5yzg/PbonPHxErS9OnTOfmkE3h9xNtUVVVx6CEH8dyzz7DDjjtlHW2BNlpnNY7YZ0u2OvQKAO695lj23aEb5x6/J8df+DdGvP8Ze/bckGt7Hcwhv++XcdrGDR70MnvvvR8jR4zIOkqTldt7J1c5Zwfnz5SvY2JWXENeGUynTp2pqqoCoEePrXni8QEZp2rYWqtX81lOJWTUZxPZYYt1+HGn6nkVklGfTWT7zdfOKmKTHHDgT2nfoUPWMZql3N47uco5Ozi/FYY7Js0gaRVJ10rqLamPpHsl9W3kNafm3L9O0iRJP08frydpvNIBQkm9JP1XUpdmZNu/7nWSNpT0rKSjm9pO1mpra+f7YOzYsSMTamszTLRww97+lHXXWJmq1kkRctP1O9GhfRsGvfEJW3TtAkD3DbuwdNvWtGzpX7tiKrf3Tq5yzg7OnyWRzDEp1m1x8lBOE0lqAzwCHBgRY9NlrYF7GnnpqcAN6f0zgaOAp9PHewLfApsDQ4H3gIERMboZEfcHvgFGR8Rbkl5sRhuZq6mpYdrUqfMeT5kyheqamgwTLdyYcV9z0qX3cs6v92DipKm88/E4Jk+dwTnXPcSJh23PWqtXM+XbmYybMJk5c+ZmHbeildt7J1c5Zwfnt8Lwn25NtxfJh/7YugURMTsifgog6WJJl0u6RNJZ6bJDgGXT6sqhETEHeDJtC2BV4C5g7/TxVsDg9LUHSLo9raLcmnaMkPSntL1rJJ2eLtsC2Bg4WtLZOZm3lXSbpEGSuqfrdpR0h6RzJN0iabd0+ZWSxki6SNKTkm4gA1tu1YMxYz5l1qxZAAwePJA99tyrkVdla9Lkb+lzy6PcfM/zLNO+Lfc98SorVy/DRbc+xi33Ps8Xtd9wz4BXs45Z8crxvVOnnLOD82ereNUSV0xK35rA+LoHkjoDhwFbA48BW0XErulzz0v6b0TcL+mqiOiT085jwCGS/kVS4Xgc6AdcACgi5khaDrgVWDMiZkjqAxwP3Ag8FhEPp9sZLqlfRAyVNBzoHxHP52xrfEScJ+mnwC+AYcA5wEcR0VdSW+A9SWtFRC9JvyOp7kwButY/AJKOA44DWL1Tp2YfyIVp164dN918G6efegorVlfTtWu3kp+Adm2vgxn4+sfM+u47BrzwJu+P+pJex+7GSYclZ+V07NCGy/s9kXXMvLz04gvce8/djB8/jiv7Xsopp55O27Zts46Vl3J879Qp5+zg/FmrlMmvioisM5QVSQcCh9dVSHKWjwb+AuxG0ukA2AC4KyKekjQ6IrrkrL8cMAr4LfAhyRDO58AvgRUj4u+SNgceBm5KX/YjYExEXCPpBKATSefhOGDHiBgtqT85HZO0M/NpRNwpaXvg6Ig4WtIA4CvgnbTtbYBjIqJW0qcR0Tmf47HZZt1j4JBh+axakpbb/LdZR2i2r4b8IesIi6RSvtfDrKnaLqXXIqJ7Idts96N1Yu3jbi1kk/MZcdHOBc/cEFdMmm4AcJ6kLnVzQNJJqy2ByUBtRFyRLt8R+Ch93Zx0vW4RMSIiJkkaSdIx+UlEhKQngGuB7dLXfATMBK6JiO8lrQX8SNJGwFkRsWa6nX1z8s1JI60FfJkuW1DvcwRJJeWmtI2fk3RUGlrfzMxK2OIecikWd0yaKCJmSdob6CVpEkmH5MdAf5KKSUclZ+hMBZYD6uZ6DACuSe+fnv77GLBBRMzNWWfdiPgq3dYkSb8HbpQ0FugMXETSAXpX0p9JJsquChxDMgz0FHAsSefiKqAn0FXSk8CRQLd0nklf4CpJ5wOtgXHp8NGxwDKSTouI6wp24MzMzPLgoRxbJB7KyY6HcszKU1GGclZdJ9Y9/rZCNjmfNy7cabEN5fisHDMzMysZHsoxMzMrc3UXWKsErpiYmZlZyXDFxMzMrAJUSMHEFRMzMzMrHa6YmJmZVQDPMTEzMzMrMFdMzMzMKkCFFExcMTEzM7PS4YqJmZlZuVPlzDFxx8TMzKzMJRdYyzpFYXgox8zMzEqGKyZmZmZlTxUzlOOKiZmZmZUMV0zMzMwqQIUUTFwxMTMzs9LhiomZmVkF8BwTMzMzswJzxcTMzKzcqXLmmLhjYku0Sa/enHUEMzPL4Y6JmZlZmUuu/FoZJRPPMTEzM7OS4YqJmZlZBXDFxMzMzKzAXDExMzOrABVSMHHHxMzMrBJ4KMfMzMyswFwxMTMzK3cVdIE1V0zMzMysZLhiYmZmVuaEPMfEzMzMrNBcMTEzM6sAFVIwccXEzMzMSocrJmZmZhWgRYWUTFwxMTMzs5LhiomZmVkFqJCCiTsmVrqefeZpHn7oQaprapDEeb0vzDpS3so5Ozh/lso5Ozj/kkzSOsBhwAxgO6APUAv0Bj4CugCnR8S0hbXjjomVpOnTp3PySSfw+oi3qaqq4tBDDuK5Z59hhx13yjpao8o5Ozh/lso5Ozh/lqRsvytHUkvgOmCfiJgr6S7ge+Bu4IKIGCrpZKAXSUelQZ5jYiVpyCuD6dSpM1VVVQD06LE1Tzw+IONU+Snn7OD8WSrn7OD8S7jNAQEnSzoH2Af4BtgBeDVdZyCwV2MNuWNiJam2tpb2HTrMe9yxY0cm1NZmmCh/5ZwdnD9L5ZwdnD9rLVS8Wx46Az2A/hHRF+gJnAHMiIhI15kC1DS6H83b/fIhaQtJz0saJKmPpD9K+qek1k1sZ0NJz0o6ut7y1pJukDQ9bf9SSY9L6tHMvKc253WFJGnZ+vu5uNXU1DBt6tR5j6dMmUJ1TaPv55JQztnB+bNUztnB+bMmqWg3YEVJw3Jux9Xb/BTgvYiYnD5+GdgQaKsfxpg6ksw5WaiK75hExFDgeWBQRPSJiBOAFYHdm9jOW8CLC1g+G7gBmJ62fz7wJ5JJP82ReccEWBY4OssAW27VgzFjPmXWrFkADB48kD32bLQCWBLKOTs4f5bKOTs4f4WbGBHdc2796j0/BFghnWsCSQXlbeA5kmEegK2BRsfGlrjJr5JaACsAEyRtDfyCZLbwusD5EfFFQ8tz2lgZeAp4HLh2AZtZkbRXKKkVScelFlgGGB4Rd6fbOBL4ENgCOAE4GFhWUh/gFeADkslEg4CuwLURMVzSCcDZEdFF0q5AP2B7YCngTmAcMJ5kjG9/4CTgc6A9MC4irk0rOrcBrwOzgY2AkyNiGHAc0CXN8Z+IeKUZh3qRtGvXjptuvo3TTz2FFaur6dq1W1lMQIPyzg7On6Vyzg7On7UsTxeOiK8l9QJukDQBqAYuBu4BLkg/qzoBpzXWln4Y+qlc6QfsjiQdiU1IPoiPBT6agY7ZAAAgAElEQVQGNomICZJ+BuwHHAGMrb88Ig5P2/kUWB14OiIGpe13IekZXkLSgWgB/Doipkk6HtgsIo5Ly1nvkoy9nQt8BVxGUu76OCK+lTQ6Irqk7a4G1ETE65I2Bc6JiIPT53LXex44OiJGp0Mwe0bEIZLWSrexXUQ8nK47HNg2Iqam+1MVEedIOhjoGREnp/vTPyK2b+B4HkfSeWH1Tp02++DjT5v8MzEzW1K1XUqvRUT3Qra5TOf1Yptz/1rIJufz+AlbFjxzQ5akisnQiLgCIJ0xfB3QMSImpM9/RFI1WLGB5XWOIRnqeLJe+zMi4gpJVSQzkNcCRgDdgFUknZ2u9xawMkmH5DxgKDAYOGsBmb8DDpW0B8nYXHWe+/ouQER8nO7vKpIuJxkD7EhSMaobSP0g/XcC0IE8pCW8fgCbbda98nu2ZmYlToCojCusLUkdk1zjgE2ByZJqIqIW+D9gODCxgeV17iDpTNwvaZuImJTbcETMSjsBlwD7knROZuV0ig4ARpNUMU5Nqyj3AXsADwJz0/U2IpnnMSkiLpO0NrBlA/uzer3H8zoLaTtnRcSa6eN9G1o3xxyS9zmSNoqIEQ1s18zMrKAqvmMiqTvJ0ElrSeeR7HNX4AJgOaCvpI+BdYAzIiIkHVp/edox6AlMAl4iOT/7QUkXkAwBtZV0RkRcA9wP9JbUF7gUuFjSRSQXmyEiHpLURdL1JHNPZpBM0AV4LX3dNOCBNEcV0BroLGmniHgGuFvSH0iGkKYCJ6Tt7QMsl7PeB8C7kv4MvAesChwj6W/p/nSV9CTJfJdu6fEaDsyUdDXwPknnyszMSliep/WWvCVijokVz2abdY+BQ4ZlHcPMrGwUY47Jsp3Xi57n3VXIJufz6PFbeI6JmZmZ5emH642UvYq/jomZmZmVD1dMzMzMKkCFFExcMTEzM7PS4YqJmZlZmRPQokJKJq6YmJmZWclwxcTMzKwCVEjBxBUTMzMzKx2umJiZmVWASrmOiTsmZmZmZU7yUI6ZmZlZwbliYmZmVgF8urCZmZlZgbliYmZmVgEqo17iiomZmZmVEFdMzMzMKkClnC7siomZmZmVDFdMzMzMylzyJX5ZpyiMJldMJO1WjCBmZmZmDVZMJD0HRP3FQCdgrWKGMrPGzZ1b/9ezvKyw5clZR1gkk169OesIZj+QKmaOycKGcoYCty5g+YlFymJmZmZLuAY7JhHRq+6+pBbA8sBXucvNzMysNFRIwaTxOSaSdgU+Ae4ADpN0fNFTmZmZ2RIpn8mv+wDrAgMj4h7gR8WNZGZmZk2ldJ5JMW6LUz6nC38WETMl1c20m1XMQGZmZtY0lXS6cD4dk7UlnQ2sK+m3wKpFzmRmZmZLqHyGck4FOgIrAisDnvxqZmZWYpaYoZyImCrpapJrl3wcEdOKH8vMzMyWRPmclXMk8DbJWTnvSDqq6KnMzMysSVTE2+KUz1DOvkCXiOgGrAHsX9xIZmZmtqTKZ/LrqxExGyA9O2d4kTOZmZlZE0jQokKusLaw78qpG7JZU9LFJBdZWxN/T46ZmZkVycIqJr8AXgTGpY87A3NIzswxMzOzElIhBZOFdkxOiYi36y+UtGER85iZmdkSbGFf4jevUyJpPaA6fXgk8Osi5zIzM7MmWNzXGymWRie/SroKWIfkO3I+ANYrdigzMzNbMuVzVs6MiNhPUq+IuFLS6UVPZQY8+8zTPPzQg1TX1CCJ83pfmHWkvJVzdoDx48dzcZ/zeXPkSF4aNDTrOHk5+Ygd+FHNsnw7YxZVrVvR+6ZH2Gid1Tjp8O1575PxrLfWKlx862OMHT8p66gLVe7vHefPToUUTPLqmLRO/11OUitgsyLmMQNg+vTpnHzSCbw+4m2qqqo49JCDeO7ZZ9hhx52yjtaocs5eZ/Cgl9l77/0YOWJE1lHystE6q3HEPluy1aFXAHDvNcey7w7dOPf4PTn+wr8x4v3P2LPnhlzb62AO+X2/jNM2rNzfO85vhZDPBda+k7QPMAyYCswubiQzGPLKYDp16kxVVRUAPXpszROPD8g4VX7KOXudAw78Ke07dMg6Rt7WWr2az3IqIaM+m8gOW6zDjztVz6uQjPpsIttvvnZWEfNS7u8d58+OEC1UvNvi1GjHJCIuiIhHI+JfJKcK/6b4sSqLpC0kPS9pkKQ+km6U9E9JbRbymo0lbZ9H22dJCkmXS7pM0n8l7dnMnCdKGt2c1xZabW3tfB+MHTt2ZEJtbYaJ8lfO2cvVsLc/Zd01VqaqdVIE3nT9TnRo34ZBb3zCFl27ANB9wy4s3bY1LVvm8/dYNsr9veP8GVIylFOs2+K0sAus9WzgKZ+V00QRMVTS80D7iOgDIOlh4ADg3gZetjHQBXi+kbavknQlcHlETJO0HXA98Hgzct4q6aymvq4YampqmDZ16rzHU6ZMobqmJsNE+Svn7OVqzLivOenSeznn13swcdJU3vl4HJOnzuCc6x7ixMO2Z63Vq5ny7UzGTZjMnDlzs47boHJ/7zi/FcLC5phcD4zgf7+/p2vx4ixRVgBqJf0G2AAYT9IROQFYnuQ7iZaV1Af4Y7p+L+BNYF3gsoj4ZAHtVgMTACS1BK4BvgKWA96PiH4NLc9tRNIZwF7AbRFxf+F2Oz9bbtWDMWM+ZdasWVRVVTF48ECOP+HExR2jWco5ezmbNPlb+tzyKAB/ueQobr//RVauXoaLbn0MgJ22Wpd7BryaZcRGlft7x/mztSScLnxKRAysv1DS1kXMU+m2kNQb2APoFxHPSGoH3B4RcyXdBOwWEQMk/ZvkyxP7AEh6CDg9IgalQzzXklRc6pwmaam07TPSZccCS0XEpWkbb0l6Cei5oOUR8W76eAOSDs6uEfFdEY9Hg9q1a8dNN9/G6aeeworV1XTt2q1sJqCVc/Y6L734Avfeczfjx4/jyr6Xcsqpp9O2bdusYy3Utb0OZuDrHzPru+8Y8MKbvD/qS3oduxsnHZacldOxQxsu7/dE1jEXqtzfO85vhaCIyDrDEiGtfLSPiDPS4ZYrgR7AjiSdiYnAzsDfIqK/pKOZv2PyLXA1MBNoC6wZEUemzwXQIR3K6QC8A2wJnAd8HBHXpes9CtwFbL+g5RHxT0lTgQFA24jYr4F9OQ44DmD1Tp02++DjTwt2nCx/c+eW9+/uCluenHWERTLp1ZuzjmBlqu1Sei0iuheyzZofbxg/u/qfhWxyPjcfuH7BMzckn9OFrcAi4gVJ35BUPP4CbBQRYyR1zFltDiBJywEdSIbVHoyIkZKqmL9akmsa8D3JEM0IYKOc59YkGQparoHlAF9HxKGS/irp9xFx/QLy9wP6AWy2Wffy/nQ0M7OS4o7JYiCpO8nwSWtJu0TEU0Af4K/AN8Atkl4mqaCsI2kAMBQ4gqRKcgvwK+B0SR8BqwD/TNs+Ld3MWZJmAasDf42ItyW9B1ybVmuWA26MiPckfdjA8mOBjpIOJqma9JfUPiIuKeoBMjOzRSKWjDkm86TXMekIDAdGRcT0oqaqMBExjGTIJnfZKySX+s91Zc79CcDu9Z4/dgFtXwdc18B25wCnNmH5n4E/5yxa7JNezcxsyZbvd+VUk1xY7R2gL/C7IucyMzOzJmhRGQWTvK78+k1E/BL4JCLeAMYVOZOZmZktofIZylkx/bdukmN1kbKYmZlZM1VKxSSfjskHkt4B5kr6KXB7kTOZmZnZEqrRjklE/FHSCyRXJ30zIt4vfiwzMzPLV/KdNpVRMsnr26wi4t2I+FdEvC/pZ8UOZWZmZkumfM7KGcUP80tEctrwP4oZyszMzJpmSZpjcnlE/AlAUieSC4WZmZlZCamQkZzGh3LqOiXp/TFA56ImMjMzsyVWPkM5d+Q87Eie81LMzMxs8RDQokJKJvkM5Qjon96fSnJZejMzM7OCy6djMhOYFBEjix3GzMzMmqdShjPy2Y81gDeLHcTMzMwsn47JYKBD3QNJ//OttGZmZpat5CJrxbktTvl0TI4BxksalV7TpHeRM5mZmdkSKp85JvdGxNl1DyT9qoh5zMzMrIkkVf5ZOelpwi/kdkoAIuIvRU9lZmZmS6SFVUy+ioi/LrYkZmZm1mwVUjBZ6ByTWNBCSUcXJ4qZmZkt6RZWMTlO0k/rLav7Er/+RUtkZmZmTbYkfInfg8C19ZYJ+HXx4li5mT57DsNHf5N1jGbr1mmZrCM02+zv52YdYZF8NeQPWUdYJNtc8VzWEZrt2TO2yzrCImndqlIuJWYLsrCOycSIeLv+Qkm9ipjHzMzMmqiSvitnYd3OrST9ov7CiJhZxDxmZmbWDJVygbUGKyYR0XNxBjEzMzPL5wJrZmZmVspUOZNfPYPIzMzMSoYrJmZmZhVAVEbJxBUTMzMzKxmumJiZmZW55HThrFMUhismZmZmVjJcMTEzM6sArpiYmZmZFZgrJmZmZhVAS8Al6c3MzMwWK1dMzMzMypzPyjEzMzMrAldMzMzMyl0G3wJcLO6YWMn47NNR9Lv+UtbeYCMmjP+Cjssuz69OPmve8/1vvYb7+t/Gf4Z+nGHK/IwfP56L+5zPmyNH8tKgoVnHaZYZM2aw83Y/YcedduaSvldnHSdv5XjsZ04Yw9dvPkuLVlVMHT2CH+34C8Y8egMtWrebt87sybV0Pe3vGaZs3Ny5c/nZQfvSffMtmT17NqM++Zhbbv8Lbdu2zTpa3p595mkefuhBqmtqkMR5vS/MOlLeWlRIz8QdEysZUyZPYue9D6LnznsCcNjuW7H1Druy7oYb8/qQl5ky+ZuME+Zv8KCX2Xvv/Rg5YkTWUZrtsot6022jjbOO0WTlduxj7hzG/uc2fnzEZahFC1bYZFdo0ZKVtjmU5bvuAMDUUcOZ9umbGSfNzxZb9uDMs88D4LCD9+fRfz/IIYcdkXGq/EyfPp2TTzqB10e8TVVVFYcechDPPfsMO+y4U9bRliieY2IlY/1um87rlADMjbm0aduOryfW8vSABzn4yOMyTNc0Bxz4U9p36JB1jGa7756/sWWPrencpUvWUZqs3I79t5+/DxHUDnmIcS/cwzfvDaZVu2XmdUoAJrz6KNWb75thyvy0aNFiXqfk+++/5/PPP+fHa6+Tcar8DXllMJ06daaqqgqAHj225onHB2ScKj91k1+Ldcsrg9RW0khJ16SP20i6WdI5ku6QtHY+7RStYyLpBEm3Seoj6S5JVxRxW6tLekBSnwaev1DS9WmW+yWdWqwsjZG0jqT7FuH1u0oaLunxdH/qju/+Bc65v6QuhWyzKZ7/72Nstc2OdFrjx9x27SWccPoFWUVZ4rz37jt88P677LPfAVlHWSLM/uZLvh37DitushurbHc400aP5Ks3npz3/Kyvv6Bl1dK0WnqZDFM2zdNPPckhB+7D7nvsxaabdc86Tt5qa2vn69R27NiRCbW1GSYqO5cCb+Q8PhUYExF9geuBv+TTSFE6JpI6ApcAJ0ZEH+AYYNlibAsgIsYCjzaQZUNgv4j4fZrleGDVYmVpTES8Dxy2CK//LzAceDwi+qT7dDHwQWESzrM/0KXAbebltVde4vVXXuJ3513O+2+PoFWrVvz7vjt58J6/MGvmTO66/XrGji79eSbl6rFH/02bNm24/poreWXQQF577VVuu/nGrGNVrJZV7WhT3YmWbdoD0L7zhkwb/cMwVO0rD1GzVXl1EnfeZTcefOQJPh09ij/dflvWcfJWU1PDtKlT5z2eMmUK1TU1GSZqGql4t8a3rSOBgcConMV7AYMBIuJNYKO0f7BQxZpjMouksnSapL9GxETgBEmtgBuAWmAZYHhE3C1pTeA6YBDQFbg2IoZLOhO4ELgA+AnQEjgKuBb4BFgJ+DoiLkm3u4GkPwCbApdHxABgErCWpKOBByJiEnAmgKQO9dsC7gPuBMYB44F9SD6k1wB2T9ftDJwWETMlHVB/ObAJcBvwOjAb2Ag4OSKGSTolXaeLpF8CfYHb09euCewdEVMkbQpcBAwD2pB0Zk6JiEdyD7SkrYB1I6K/pOuAg4AjgY+AfwB/B+5Jj/uHwGrAIxHxpKQrgcOB/sD6wOSIOEbSFsDGwNGStoqIolW76hv43JOMGDaY3/e+gom14/n+++/odcn1AIz7bAyP/utvHHX87xdXnCXSGWedO+/+rFkz+XbaNH7z299lmKiyLb3aenw/fQoxdw5q0ZLZ33xJ1QqrATBn5rfM/uZL2q60RsYp8/Peu+/w6ehR7LbHXgB07rIGo0d9knGq/G25VQ/GjPmUWbNmUVVVxeDBAzn+hBOzjlUqVpQ0LOdxv4joByBpfWC9iDhXUrecdWqAqTmPp6TLpixsQ0WpmETELGA7kg+3dyW9LGl34FdA64i4GDgDOE9SDcmH98URcRVJuee8tJ2rgYnAfyLip8BlwLnARxFxRUT8Hvg8Z9OzI+Jk4HfACWkbnwN7APsCn0p6Mv3gZUFtRcSHwJ+Tl8bJwE7Ap8CtwKkRcSVJx+p4ScstaHlEDAb+DXwZESeQdLp+kea5Kec43Qm8BwyKiKOBN4Fd0qdvBy6NiIuAZ4DR9TolB0m6gaRUVufM9Fh+SNKxeisi/gick+5n3/S495PUKiJ6AdXANRFxELClpBUiYihJVab/gjolko6TNEzSsG++nlj/6WZ7763h9D71V7w1fBgn/Xwfev3mCMZ88hEAYz/9hH/+7U/MmjmTO2+5hhnTvy3YdovhpRdf4N577mb8+HFc2fdSZsyYkXWkJnvk3w8y6OWXePXVIfzr/maPPi525XbsW7XryKq7/pqxj9/CF8/dxXfTJ7PST34KwMTXn2CFzfbIOGH+WldVcVf/O7j6isu4/JI+vP/eu5xy6ulZx8pbu3btuOnm2zj91FPoc8H5dO3arYwmvooWRbwBEyOie86tX87GDwBmSjob2AbYIp0yUQvkTvjqmC5bqKKdlRMRbwNHSmoJHAg8CPwVWCUND/AWsDLwJXCopD1IglfXa+7dtM1hki4iZ5wqIu7IWe+j9N+J5ByMiBgEHCipCjgOGJB2iLotpK26bX4saXMggN+l30WwPDAN+HEDy+vUDa9MYP4fTn0LWm8Dkg4GJNWY+h6IiJvTstjSadY5kv5MMlw1GPhvum434Kuc4/5mmrWWpPM0ud72v1pIVtI3ZD+A9bpuEgtbtynW3XBjnh3x2QKfW73zmpxy9iWccvYlC3y+1Gzbczu27bld1jEWyb77H8i++x+YdYwmK8djv9z627Lc+tv+z/K6Dkq5WHPNtfj7Px7IOsYi2WnnXdhp510aX9HmiYjL6u5LagO0j4gb0vs9gJckdQVGRMRCqyVQpI5JOmmyd0T8Kv2wfIikAjAamFH3V3g6DDKaZMhiUkRcls7a3TK3vYjI/fAbAayVvl7AcRFxe92qC8iyPbBtRFwSEbMk3UMy/6UpbX0EzCSpLHwvaS3gRwtZzgLaWJgFrfcOsDbwCskQz4JfmAz7tJe0SUS8QVLteTXN8Zuc/RxfV62R9HN+6Hw0lHFOsqrWIum8TGtgPTMzy5jI/gJrkg4CegKtJR0G3AhcI+l8kj/kf5VPO8WqmEwGVpB0fXp/DaAXcAdwVVr1+B4gIh6S9ADQN61otAY6S9qJ5K/6ZdKdujIiviOZk3GtpPNIJtQ+IWklkrkgy0n6MfDznDY+JSkrXUnSiVgb+EVEhKTG2topIp6JiEmSfg/cKGksyXyQixpannauegJdJT1JMuejm6TuwBbpPh1GUtnpDBwjqX/OawaQDEVdImkoyZydAEj3qVt6fOsmFP8IGAq8ERFfSXqeZOhmTvp83/S4n58e33Fph/HYNMtBJPNrOpNMVL4AeAo4Nt1u+Zyna2ZmmYiIB4D6JbOTmtqO5i9GWKmQtAnwZlqJ6QkcFRHHZp2rvvW6bhJ3PvRc1jGarVun8jkFs77Z38/NOsIiad2qvC+j1POq57OO0GzPnlFeQ131lft7p+1Sei0iCnoedef1usU5dzzS+IrN9JufrFHwzA3xlV9L14YkE2w/JqnyXJRxHjMzs6Jzx6RERcTdwN1Z5zAzs/JQKd+VU971MDMzM6sorpiYmZmVuVI4K6dQ3DExMzOrAB7KMTMzMyswV0zMzMwqQIUUTFwxMTMzs9LhiomZmVmZE5VTaaiU/TAzM7MK4IqJmZlZuROoQiaZuGJiZmZmJcMVEzMzswpQGfUSV0zMzMyshLhiYmZmVuaEr/xqZmZmVnCumJiZmVWAyqiXuGJiZmZmJcQVEzMzswpQIVNM3DGxRdOudUs27rJs1jGWSG1at8w6whLt5bN3yDqCWQ75AmtmZmZmheaKiZmZWZnzl/iZmZmZFYErJmZmZhXAc0zMzMzMCswVEzMzswpQGfUSV0zMzMyshLhiYmZmVu7kOSZmZmZmBeeKiZmZWZnzdUzMzMzMisAVEzMzswrgOSZmZmZmBeaKiZmZWQWojHqJKyZmZmZWQlwxMTMzqwAVMsXEHRMrXc8+8zQPP/Qg1TU1SOK83hdmHSlv5ZwdnD9L5ZwdnD8ryenCldEzccfEStL06dM5+aQTeH3E21RVVXHoIQfx3LPPsMOOO2UdrVHlnB2cP0vlnB2c3wrDc0ysJA15ZTCdOnWmqqoKgB49tuaJxwdknCo/5ZwdnD9L5ZwdnD9rUvFui5M7JlaSamtrad+hw7zHHTt2ZEJtbYaJ8lfO2cH5s1TO2cH5rTBKomMi6QRJt0nqI+kuSVcUcVurS3pAUp8Gnl9b0o2Seku6TNJDkg4r4Pa3l7RxzuOLJe3bjHYOlzRN0tXpY0kaJ2n99PHekgZJ2r4JbW4o6VlJRzc1T6HV1NQwberUeY+nTJlCdU1NhonyV87ZwfmzVM7ZwfmzpaL+tzhl3jGR1BG4BDgxIvoAxwDLFmt7ETEWeLSBLCsC9wK9I+KSiDgP6A0cXMAI2wMb5zy+MCIeaWojEXEP8AwwLF20GfAtsHf6eBDwaEQ834Q23wJebGqWYthyqx6MGfMps2bNAmDw4IHssedeGafKTzlnB+fPUjlnB+e3wlBEZBtAqgI+B/oCf42IienyVsANQC2wDDA8Iu6WtCZwHckHb1fg2ogYLulM4ELgAuAnQEvgKOBa4BNgJeDriLgkrQjsBYwHNgUuj4gBkk4HVo6IM+tlrAa+J+m0CHgP2JGk0/IVSWfqHWAd4OyImCjpEqA1MBtoExFnSlob6Ad8AwwH7gBuSu8/BtwDDIyIX0r6BXA8cDiwCbB7uh+dgdMiYqakXwM9I+JISeelGX4fET3TKs97EfGGpPWA04EPgHWBq9N9X9D+bAyMBv4DPAU8nh7jBdYzN9usewwcMmxBTy2yZ55+ioce+BcrVlez1FJLlc3seCjv7OD8WSrn7OD8+Wi7lF6LiO6FbPP/Ntg4brz/v4Vscj57bbhSwTM3JPOOCYCkDYCzST583wcuJfkA3iwijlPyBQDvAj1JPuxrIuJ1SZsC50TEwWk7o4E9I+IdSd2Bg4BJEXFV+vwxEXFH2jHZJSKOSNe7MCL2kXQb8FHE/7d3nuFylVUbvp9ACBCSkEBCkQ4ivQWkiAICKor0Jk2Q3osoTSR0CCBIFZCidKRJ+aT3IkiX3ov0QOgkQPJ8P9Y7MIR0zpw9M2fdXLk4s2dm7zVtv2uv8iwfPQY7lwcG2/6+pOkJ5+de4Pu2Xy37Xdj2bpJWt/3P8rwrip2PlRTSi7bPKvdtBsxme5CkLYFFbO8oaUPgf8B/CYdjDtuflucPtf1nSTMCDxNO15HAnsDrwNzAnrb3Kse4G9jJ9n2SlgSOsb3MGF7PVsBLwMzADbbvGs37sDWwNcDMs8wy8OnnXhr7B5wkSZJ8STomY6cp2oVtPwZsImkSYC3gUuBvwAyS9ioPexSYHngT2EDSKkBvoP8ou3ui7PM+SQcAp9cd54y6xz1b/j8EqFU7/a8cY2zU9v9GiaT0K7ZT/h5RHjeZpMHAu8B3RmPn6DgX+IOkvYlIyLaSlgAM7FJ3jI+KDa9JegVYHXjD9heSbiSiQSPr9rsQEW2pve6FR/d64MshULV02rWjM9L2qUTkh4EDF6/es02SJOnipI5JByJpNqKmYwvbIyRdBpxCpBM+tX14edyaZdsBRMTgkJIaWbJ+f/56COhhYM7yfAFb2z6l9tDRmHMG8C9JvWx/WJ73Y2A929uO5nlDgLeBU2wPlTQNsLSkqYGzgd62P5O0UN1zRhRzZgQ+H8X2TyX9AzgeuL1sfhYYBhxVHI85gRnrnnYVka6qFdBeDewL7D2a9+Fd4LtE6ujLw47hfbgXuEjSsraHjuYxSZIkSdLhVO6YAO8D00g6pvw9O5GSOAMYXKIeXwDYvkzSJcBhpTZlMmBWSSsSkYQ+kv4AHGH7c6Ju5ehSfzE14XRMB/wS6CtpLmDj2j5s3yhpfeBASe8BUxDRlN+X420CLCRpbduX2LakDYAjJD0PzEI4Fe8DFwF/l3QfMB8RVbkTuBXYnajp+EOdLfPZfhw4CbgH2K685qGSdgP+XKIjsxLOWY2rgF+XwlWAfxGO3Q11j9mivIZniDqYLUf3eoqj9yNgKOEYvQdcKmlv2/+egM80SZIk6Uwq0BtpFE1RY5K0Lo0sfk2SJGlHGlFjMvcCi/j4i67vyF1+jZ/NP6Br1ZgkSZIkSfLtaJeISeU6JkmSJEmSJDUyYpIkSZIkbUBnK7Q2inRMkiRJkqTFEdCtPfySTOUkSZIkSdI8ZMQkSZIkSdqAdknlZMQkSZIkSZKmISMmSZIkSdIGZLtwkiRJkiRJB5MRkyRJkiRpA7LGJEmSJEmSpIPJiEmSJEmStDipY5IkSZIkSdIAMmKSJEmSJC2PssYkSZIkSZKko8mISZIkSZK0OkodkyRJkiRJkg4nIyZJkiRJ0ga0ScAkHZMkSZIkaXWiXbg9XJN0TJIkSSaCkSNdtQkTTbcWF7zou8SOVVsi85sAACAASURBVJuQNJB0TJIkSZKkDWhtd/Mrsvg1SZIkSZKmISMmSZIkSdIOtEnIJCMmSZIkSZI0DRkxSZIkSZI2ICXpkyRJkiRJOpiMmCRJkiRJG9AmMiYZMUmSJEmSpHnIiEmSJEmStAFtEjDJiEmSJEmSJM1DRkySJEmSpB1ok5BJRkySJEmSJGkaMmKSJEmSJC2OaB8dk3RMkiRJkqTVUbYLJ0mSJEmSdDgZMUmSJEmSNqBNAibpmCTNy0033sA/L7uU/gMGIIl999u/apPGm1a2HdL+KnnjjTc4cNAf+O8jj3D7XfdWbc4E04rv/U4brcCMA6bm40+H02OySdnvuCtY+HszscOGy/Pk828w75wzcOBJV/HKG0OrNrVLkI5J0pR88skn7LTDtjzw8GP06NGDDdZbm5tvupEVfrxi1aaNk1a2HdL+qrn7rjtYddXVeeThh6s2ZYJpxfd+4e/NxEa/XJKlNjgcgPOP2pLVVliIfbb5Odvsfw4PP/U/fv6jBTh6z3VZb7dTK7Z2HLRJyCRrTJKm5J5/380ss8xKjx49AFh66R/wr/+7umKrxo9Wth3S/qpZc611mKpXr6rNmCha8b2fc+b+/K8uEvLC/4awwve/x1yz9P8yQvLC/4aw/BJzV2VilyMdk7EgqbekjyQNHM19M0u6RNKg0dy3vaQXy9+3SzpK0pmS3i9/HyXprLEcdz1JL0o6T9KB5f9bjO2Y4/FalpX0gKTly+0DJa02lsfPJmmNCT1OR/HWW2997eTcu3dv3n7rrarMmSBa2XZI+5OJpxXf+/see4l5Zp+eHpNFAmGx+Wah11STc9eDz/P9BWcDYPEFZqPnFJMxySTNvGSqof91JpnKGTsbAf8Etga2qb/D9iuSrgRmG/VJtk+S9Pty8wzbZ0paAFjB9h4AkjYf00FtXyRpe+A821dJmhx4F5gOGO0xx4XtOyQ9Urdpf9sey1NmA9YALp/QY3UEAwYM4KMPP/zy9gcffED/AQOqMGWCaWXbIe1PJp5WfO9ffv1ddjj4fPbeahWGDP2Qx597nfc//JS9/3QZ2/9qeeacuT8ffDyM199+nxEjRlZtbtMiaU7gYOABYCbgHdsHSuoHHA48D3wX2Mf2m2PbVzO7f83Ad4HdgLUlTQUgaVNJFxfHY5naAyXNLulKSQdI2rq23faZo9txcVZ6STpF0p6STpK0+hjsmBb4FBhev1HSGiUSs5ekc0uEZ3FJT0s6szzm15LukjRb3fNmAS6rRV5KhOdQSXtL+oukyYDNgEUkDZI0zwS9ax3Akkstzcsvv8Tw4fGS7777Tlb5+S8624yJopVth7Q/mXha9b0f+v7HDDrxSk447xb6TDUFF/zrP0zfvw8HnHQVJ55/C6+99R7nXf2fqs0cJ1Lj/o0H/YALbB9pexdgg5JtOBS4wfbhxIXuUePaUUZMxoCkJYHbbb8l6RpgQ0mXAEcAs9keLunguqcMBs6xfaGkuYB9xuMw+wDP2D5KUg/gOUm32a4lPDeU9H1gGmAV25/p69+QocCutt+XtDuwie0TJR0OLF4eMwL4ve0Xa8+1/bKky/kq8rI1sIvtWyUtU45zFrCZ7UHj+551JFNOOSXHnXAyv911Z6bt358FF1yoqQvo6mll2yHtr5rbb7uV8887mzfeeJ0jDjuYnXf9LVNMMUXVZo0XrfreH73nutz5wHMM//xzrr71vzz1wpvsueVP2eFX0ZXTu9fkHHrqv6o2s6mxParn1g34GPgFcEjZdifwt3HtKx2TMbM+8LGkhYlIxTbAQ8DbtmuRi+eBWcrf8wOH1W0fHxYCTgcojs5QYC6g9gGfZ/uqsTz/I+CPkoYAiwGP1Z4H7CepD7Cs7XPGYcdmwO8lHQmcDdw1tgeXiNDWADPPMsvYHvqtWHGllVlxpZUbtv9G0sq2Q9pfJT/80XL88EfLVW3GRNOK7/3KWxz7jW1H/PXaCiyZeETDm3KmlXRf3e1TbY+2TUnSmsC1tp+UNACo5fc+APpKmtT2F2M6UDomo6Es6O/ZPrBu23NE9GE6SZPbHgbMAdTe3MeBuYn82hzjeaiHgTnL/icH+gLPTICpfyUiHbcVZ2FGANvDJJ1b7r9yPPYzs+0NJfUEHivPHRFmaXJgVttP1R5cvoynAgwcuPjY6lSSJEmS9mCI7cXH9SBJKwArALuWTW8BvYD3gN7A0LE5JZCOyTcoKZWTgc/qts1DFJ8eCQwCLpD0IOFUTFcKW/cEji8RlveAPpI2sn2upCmICEMfSb+xfUbZ9WHAnyT9gYi87GD7PUlrA7MC60t6w/Z9xY6ZgV8SHud8RLRlP0k3AwPL9rlsPwucBNxDFPAiaVkiQrOJpBdG2c9qkhYDDFxq+11JjxEFTEcDVwFfOiZJkiRJE1KxjomkXwA/BHYBZpA0K3A1sDTwCvCDcnvs+xl7Y0bSipTi1f7A5rYPHtfjvw0DBy7uO++5b9wPTJI2Y+TI1j13duvW2kpcfZfYsWoTvhXDHjrx/vGJPkwI8y+0mM+7+taO3OXXWGSW3mO1uRS63grUFoSewInAFURt5kvExfxe4+rKyYhJmyFpSiJ98wKwV8XmJEmSJJ1EZ+uN1GP7fmCqMdy91YTsKx2TNsP2J0Dzl8EnSZIkHcp4tvU2PaljkiRJkiRJ05ARkyRJkiRpA9okYJIRkyRJkiRJmoeMmCRJkiRJq9MJCmudRUZMkiRJkiRpGjJikiRJkiRtQJXtwh1JRkySJEmSJGkaMmKSJEmSJC2OSB2TJEmSJEmSDicjJkmSJEnSBrRJwCQjJkmSJEmSNA8ZMUmSJEmSdqBNQiYZMUmSJEmSpGnIiEmSJEmStAHtomOSjkmSJEmStAHZLpwkSZIkSdLBZMQkSZIkSdqANgmYZMQkSZIkSZLmISMmybfigQfuHzJFd73UwENMCwxp4P4bTdpfHa1sO6T9VdJo22dtyF7bJGSSjknyrbDdv5H7l3Sf7cUbeYxGkvZXRyvbDml/lbSy7e1AOiZJkiRJ0uKI9mkXzhqTJEmSJEmahoyYJM3OqVUb8C1J+6ujlW2HtL9KWs92tY+OiWxXbUOSJEmSJN+CBRdZzJddd2fD9v/d6aa8v7PqbjJikiRJkiRtQJsETLLGJElaCaldgrVJ0vWQlGvueJBvUtKWtMsCXjuRSZoSwJl7bRh173WeF8eApEmqtmFCaIbzgKTvSVq53JxP0uSNO1gD/3Ui+QNM2g5Jqi3gkqar2p6JRVI32yPLazhd0qpV29SRSOpf/t9DUuVp5fJeTw9sKmmqqu1pNiR9D/hO+bwWq9qe8cG2JfWWNEsVxy+O0WfAOpL+Ccxue1gVtrQS6ZgkbUVZzK3gcGADSfNVbdeEUpyrkZL6AVsAVwG3VGtVxyFpZmDv4mytBgyoyA5J2rtu02HASNsfSeqW0ZOvMSewN3A4MH/FtoyR8pl+r/zdB/g/4DsV2NGtXCC9CnwM9AZmLPd1lzRTBx+xof91JvmjS9qKspgL2A24HzjF9uOSOv3E9G0ozlU/4EpgBdvnlsWypULpY8L2K8DjwPHAzLZfgzhhd7IpawCbS7qk3L4FeKfYOBKYrJPtaTpq6RDb/wdMCSwAPFru61bBZzYujgX2kLQMscbdbvvu2p2dYW/dhUU3YG5gELAHsIik3wBzActK6tGxx23cv84kHZOkLZA0W93N2YHpbP+jLmy6ZCss6qPkxPsAxwEzSjoSwPaISgzrIEaJQDwNnAesImnZsm1lST070aQrbM8NjJR0HvARsJOk40ro/bCOXjxaiboIZO1z24twlreQtCCwOrBeZQaOQqnfOI9wen8MLAbMIGkjSb+W9HNghUbWntRSyeUYOwNrAZPZvh+4AFgW+CvwtO3hjbKjlUnHJGl5JK0H/Kz8PQVRqvUzSQuUbXMCKwC9KjNyPKg7oU0taUlggO0LgaWA70v6p6S1KjZzoqmrmelTokF32d4XOB04QNIOwEJ0goxB3UI7CYDtdQEDZxJXtv8mIidndNXFo+7zEjBI0na2X7d9HPA/YCtgfeCVSg0tlN/PMOAB4L9l87LA94EewNbAEcBzjSoilzR13b63BR4BTgLWl/Qr27cCuwJb2n6gQ4/d4H+dSTomSctj+yLbf5F0KPAr288BFwFXS9oG2AY41fZ7lRo6DmpOCfA7YoG+XNI6tj8kcvtzAQ9XaePEUrfITQtcTNQpHCmpr+0LiPD7csD5tt9vsC21MHtfYENJO0nqY3sj4DLgaOBDYDvb/x3rztqUUZySPcrmDSQdBmD7cOAUYG/bt1VlZ426yM60wK3ADMD5RG3HpYQjtQmwYjk/NMKGrYnfLZImA34CLAFcCEwB7Cfpx7Y/sP1EI2xoF1L5NWlZaifP8vccxBURwIO2L5T0E6Ii/j3bD1Vl5/hQFoBJiUXxEtu3SjoKeJEofH0b6Gn7reqs/HaURWMH4CHgPsJh7A6cXeqAutv+vJNsmRrYHniGuPJ/DdiFSJ9dAmzQqAWsVSjfyQuAd21vJ6k38BdgOHCd7fMrNXAUFF1eewI32L6mbOsFHAPMB6zRyN+PpHmBJ4EtgX8B/cu/T23fLulA4rv+TCOOv9AiA33FjY1Tfp192ik6Tfk1IyZJS1J/RVeuVGayvRdwD7CgpN0Jh+SWZnZKaikFB58DbwIDJe0L/JFYxH9KnNxa2SnpCZxNREWG2X4VOAGYBzhCUq/OcErqUjjbA2/Z/gewDjA9sBmRlvhJV3VKRqm9WAb4ANhI0pq2PwD2BxYBXq/CvjFRIhR/Bn4BLF3bXqKN5xKR1Ib8fiTtJmlp4FlgVeCXwHbAZ7avAxYuFxkXN8opaTcq1w5IkomhFikhrnZfAd6UNJXtyxRiZHsDt1dm4HhSnKupgG7lxP8uEXI+l2jJ/CHwt7rX2zLUambKzWHAoUSoexlJrxPRihuBa8sC0nBb6t7HlwgHdlbbL0k6kegOMjC0kbY0K/WfV4k+PGR7K0nnAyeUiNdtwC9qXVRVUleT1c32Z5J2JBzM3SVtQbQJv2v75gbaMDdRvzYHMAK4nriYWBtYo9S8PUjUUz3SKDu+tKdNROnTMUlaitoVXTkhrQMsT4TgzwBelvQJMRl0A9uPVmboWCivYUtiUR4C/AN4StILto8pV/W9iCv4Q5thEZhQ6gt5gZWAh0s4+yngV8ABwA3AX2x/0Um29CZy/i8SDmA/YBtJTwDfpRUnynYQo3SS7E+0Sa8vaQ3bN0kaTHSILdMM38e6iGlfYM/ym3nN9rGS/ky0oS8H7EiknhqC7aeB1STtRPxez7J9r6L9fHei8HXNEiFMxpN0TJKWYZQrcIC7gA2Bg4gWwVeAHwEvN3knxRpE+qAfMC2xID4DbC1pe+Aftt+S1KPJX8cYKYtcX0JP5kPgIEm72r5W0jVEmuC6TnRKpgZ+S9S0rEQsslcS7//cwMnNsOBWRd3vahBwa3FGZgFWkvQ84UQv3wzOvqQpbX+i6OzaGrgDeB+4tDhWJxJRixNLFLKRtkxKvGfHE+eizSR9CLxARGwv60ynpHoB/o4hHZOkZagLM19HiCYdJGltYDZikV+BSAs07WJeFsrLJD0M/BqYDnjW9iOSjiGclKUkbWP700qNnQgkTWJ7REmnHQScaft+SQOBfRTaF7cDm9v+pNH2FKekJ9HpdE2J2mxAOCkb2b6yvoi6qzGa194N6FfSIvsDMwO/dHROvVKeM+oFQmfZKsIBOB+4E1gQGG77qnL/6kQtyWeSfm/7swbZMYm/0hNaEJiFqAE7RtLOhHrwS8AfGp2ibFey+DVpeuoL8srf5xOzJ3YvJ8juRE73ToeIUVOi0FXpW670PiQ0M14mBMbmIAoK7wQOblGnRMUpmRFYmFBR7S3pYNsbEBGKzYiQe8OcklIQ/WNJywPY/phITcxabl9A1ALMVZ7SZVsT6wrIj5G0KHGxug3wCZHu+gHw2CjPqer9moewq/a59QHWLik6iJbckSWK0ZBC6rrvuCQta/tBoh355+UhpwGfEsJ9ne6UpI5JknQCkmYpV72TSDoaWNT2mYTi5M4lyjADsZjfPdadVUg5WS4D7EeEfhey/SJR5GrgQOLkdlDJW7cU5SqyljI5E/i57YsJB2wuSUsRrdurOOToG8nhRFj9GEk1DY5XiaLbDSUtTizA90PXnNisr6sgz0ws9r2AI4nIyBzAYGKBbQotF9tP2P49oc6L7SuINvprSgr0h8CRtr9o1Gdat99LgKMk/YpwuBcsqcsRwI62b6y/oEomjHRMkqZF0l+BTcvNpYm0xxGSliqL+lZEpGRYZ6QFvg2lluJSIvTbD3iqbH+CaHGudUF0io5HR1OuIqej1PvY3q/c9TyhrrofcGUjnRLF3JbfAjfZ3pJwALsrJgZfADwHDCSclmNtv9koW5oZhahd7ap/U9svAxsBs9l+l0hzHQQcUKVTUlvYJV0k6UyFnDxAT0kbAdhekxDne4ZQ6f1fg2wZda08EziYOAf9nSi0ncn2Z7Zrs5Y61+Ft4JycznaxssYkaUoUbXarAldK+p7tO4A7JG1MXAkfT4TnV2rmCENd8eUktodI2gXYmOgG+SuhDvkcUFN4bSnqXl83ogNiXqCHpJttv1wWunUVOiWNfn3nA+sCk5fbixCO4DrE7JRtbX+saCv/qMG2NCWlBuI9SVcRzse2ivkyrwH9JU1TW1gJtdTKqFvY/w7sQ8zn2ZhQot1O0qK2H7R9USPt0NdVcHcjCmsftf0CcJViztPKRJvyDs1+kdQKZMQkaTrKIv4pcA6h5bG7pMXKIngOcBSwCvDfFnFK+gIbS/qh7ceAQ4j8+E7ECe2FVnRK4Mvi0j5EVOQQQljqJWAtScuXehMa/foUAluDgb8BT0r6NVFUvI7tgYS+xfzl4R830pZmpRT99gTuJt6DQUQEaRbCeduLkOiv/IK1RL82K/UjNwKXU0TLCIdqPmIGTsOpc0p2BEZS2pAVhdzYvsP2/sA1xb4KaY8qk3RMkqajruL9XELNcXJCE2CucrV7CbBNMxe6wtdaZncmFu7TJf2mXK3/jpjns2O58mpJFN03VxJ6DSMdypZXEYMHd6OBGhJ1NnQrIfT7bW9OiID9hShwRdER9DTRwtlVa0p2AvraPgzoC2wAfOJQRf6j7c2I2pyb3OAW7vHkfMIBWYb47dxDdFFtRkgDvAbsK2nyRtVySFpGUvdyczPi/TqWKLb/jLhg2ro8dn1gc2CqRtgyPoj2SeWkY5I0DZJWlrRY+bsb0RUwGSFGNoyQMN+3pAWaOlxa8veTEyfXm22fQVz5/aYU8X7f9r1uQZn5Wr5d0mTlc9iNOFkfVB7yGHAd8Nu6tECjbKkN5JuktkCVxeufwO2lLmFp4DDbbzfSlmZF0iBi0m0turg4sAAwjULNtdatdHSJ6FVKcXaPI5zd1Yjv0htECuq7tm+0vRNRQD6sEY6mpK2IuqgtSqHwPcAj5b28kBj2+B1iNg5ElHBnN/mg0FYhh/glTYGkI4miu0+Ik5KJorY9bO8p6bvECeF3tm+sztKxU0vf1N1eiuh2WMH2PqWKf29gtVLA25IoWp6vBo6zfb5CVv9S4ur2AqIQccTY9tEBNtSrf+5PiGx9Yfugcv89wADgB+6i4mklzfYTomNlBNHF0of43F6VtCKR4jrd0VZdKfq6RgiKLq+dgDWJUQGP2t6l3NcQPZXy3Z6K6Ez6CVHAfSYRlDieiMQtBlzk0B9qCh2chRcd6H/d3LjGxO/07ZFD/JIuxyXE8KvbCJn2qYlUzkaSVgHeAtZq5jY8fTV6va+kzUtR3BNE4eUiktYgWptXaUWnZJT3fRoiFfVHSRuW9NQfifbgGxrtlMCXuf9+RJrvPOBeYmDaL8tDBhMD+bqqUzKJ7fcdgwr3I5zGVYD7ilMyE7Hw3tkkTklNI2TqkkaZy/Z7xdHcgYg4LiOpXwOdkr2I9PHxtm8htHjmBLYsKa4PibqwG1xm3zSDU9JuVF7klHRtJE3q0B34d7n9GXF19Hui0GxZvsp7vw/NWyNQFsppCUXXbsCSRKj3YiK9sTmwq1twbkZdIW8voAfwpkPp8l7gIsUws+HAVp2cMpmZ+F48Q8xN+i3hnPQutUhdkrpFvhuhsnu6pMOAfYHFSyRlADEg8vFKjS2U79c0wG8IaYCekm6y/Q+HRtHdko5xg8QHJW0CfGx7E0lPKQYB/o1IJy+iaEU/hyiqbspUcnNesk04GTFJKsX2F6Ue4wRJ89q+nrgyOpAYrHZ3eUxTOtHF9t3Kgg2hB3EzMa/jA+BRYCrbvyPksluu0LXOKelDyG3vAJytmMx7JzFqfihwdqOdEkkLSlpKUk9J3yMia9sSEYGziYut5ensNoImoha5KzcPBQ6UNLft14mo1uzEZ3hdszglQE2EcHciJXgK4egOL5EdABrolBxLXBCdVzYdCvyvXDRdRagybw5M2qxOSTvRlCf7pP0ZJZc8BzAlMUztCduXKNpMTwHWAz5skk6B0fErIt88Wbk9L7AQ8B/gT0Rb46qKqbqtKjNfm8x7HNGq/QGwKPAnSScAtxLTgxsaySodEr0JgbTpgWNs3yrp74QQ3yOEbPlRtt9vpC3NjL9qb92ZeE9OAI6XtKfthyT9H/Cc7fsqNZSvfb96EhfK8xC/pz7E920e4rvWME2VEu3rSQgfLidpBJG+ubruYXcQKbCHG2VHR6A28cczYpJUgr9SnlzH9nOEgFK96ukzwG4OVcpm5nqi1mKOcvskQl9hEcLZWpwoknOzpqDGhKRpy6IxJTGH5BqiTmFDoiDxbeAIYOrOeG0OVdyniCjJCGIyMLYPBP4APAyc0FVrSkZhE2AG2+cRn9E1wOBSt3GW7durNS+oc3ovJZyBm4gaj6GEI78I4Vw10oanbW9FtJjvQTi5f3MIItY6vZ5qdqeknciISVIlZxAD7OYnBJTWl/S07WdtX1OxbWNF0nnECfQ+oujy95JGOibpbkyIMa0JXNgCztU3kDQzsZD9jbh6/Q+hV7IG8CbRafQ8cIRD3bWRttSuqlUWi32JqNSykkw4tG87Bqp1SUbtZiG62nrDlxGU24nBikcoJlcPqcLOGrXasnKzG+GQfGL7REWb/caEU3+SGzTGQHWKruW64QJJIwmnez5JPYhi10bPduo42iNgkhGTpPPQN+dNHEpEF+YmBJ8WpkQeRvPYZqMXcAxR2zAfoeExbznZPQxsTaQaXqrQxommLAY3EK2/09t+vnTe9AJWJFq5z210zUydU9KX6ABakVAtvZaIkKxM1CU0rQJwo6krdJVigjWEE7mKpJMlrUo4lIcD/6ZyddIva8tmlLQfsZxeBmxZHJajiajFoAY6JZPWpbzWKHVifRzy9scS79VgIkKYdDIZMUk6BX193kRtQuhTtm9QyIf3IdqEN5N0q+2GK4ZOKMX2+W0/CpxOzPJ5ioic3EuEny8jKvtNXLW2FHWf0yTA7USdzI6S7i2dEZ8SJ+zXOqP7pq5TY1tiQvBGwBLAX4kU2k3ACLegUF1HUIuUlO/mX4D5JV1HfD8XJVJcyxDdJMMI+fnuY9pfg20V8Tn+k0gD7k/Uk8xM1C4NKX+/4AaPMCiO0STEBcRwQoRvDUkblxq3XxDDKF9spB0dTZsETDJiknQO/vq8iReJRXyQpOVK5fs75UrpemC26iwdPcX2q4h5Iti+nEjjvAP8lHDy1yYW7ZakzimZluiKGlrqN/YDzlGoYc4GvNJIp6Rc+e8iaTlJsxFh/duJ1N/HwIzEFe2atl/vqk4JfFmrVRugeBOwFrE+rQd8j1j8jydUSncFDnGD1XjHwlSErs9QQk9lEPAzonPtGKLr5deNNEDS7pJ2LTc3Baa1fYbt9Qj11gPLfXuUi6Z2WetbinRMkoZS6i1qrAX0J6rd+xFXvMdLOlgx82J1otWzGaXDFyVOoItLOk6heXAFMKy0Ye4ArE+MQm9JilMyNRGVuAWYUaEHcgHR4fED4K+NrCkpC8FuhGT6PoRi6SNE9Omwsu024sq/8q6SqpC0Wd3NY4g24Frk6AwiPXoyoVVS+z0dafuNzrSzhqQpSxTkDiLtuQmRhnvX9nHELJp1CVHFRtmwKfAF8FyJlvwPmFTSLOUhBwFPl9TYu9C8mkmjo5FzcjrbPctUTtIwJK0N7KyYqrtNCZE+QVS+30gUVC5BzJIZJulV4IBGF1NODLYfAB5QiFRtRMzwGFS2fVJaL/9ToYkdxU8Icav/EOH11yVdbPsKSTc0MsVWnJIjgP/Y/pOkjYD+ts8uBZG7EFfYcxOiYS0nVNcRSBrA19Mxg4jPaytJj9p+UtKFwDl1NUDXd7KZXyJpIaCPpNeJyM0xhAz+CsQ042uBdxochTsJmMf2j+u2vU/UuA2T9CLRFXR9Kzkj7UpGTJKGUML+6xCiVx9IOrNciTxOFJStSqQ9DnfIzHezfZ/t5ys0+xsoxLwGlYjOpsSV+1+IUPkFRLh8E0ndWzHsW7O5rtj4Y6LNeQsi9P8QES2CxhdNHkOItdVsqelKXEEUGK9IRFMu7KpOCYDtt2yfJuksSdfZHmr7HGKW1AWSTgYecojfjTpKoAqmIPR+DiNamF8lol63ElHGQXylA9ThlN/tlMB3JO1Y22773mLTh0SX19W272mUHZ2BGvhfZ5IRk6TDKembHkTnxG+Jk89WRIh5cyIlMgfwfy7TTN2k8yZs/7t0DjxCXKW/L6l7ac3cVyFW9bRDY6OlqOt4mQb4laJF+GSiG2c2YiRAd6LQtKFhbUmTEYXDzwMrKuYMPUWkkDYkag8eBrax/Wyj7GhmVNcSrNCWOQ3YTtKVtn9p+wxJiwBXuG7KbVURgNr3y/Y9JXraH/hC0jS235H0DKFXdIrtYQ2y4QCgn+3NJP0I+KtiKvafAGzfRjhJSROR04WTDkXSEcBsttcvtwcDZk0zngAAFPtJREFU/yBOQPsStQFnEUOwmnYxr51Uy98rA3sS4eb1yxXoD2zfUamRHYCiDXdbor30t0TnxkGEsNUywKnuRB2WYs+WwEpEO/Lf6+6bvFELWLNT50SKmMH0Zi1No1C+nZNIvd1q+93672/F9vawPbyk4pYjol7/BmYi9HAubdR5QNJ8hErxCiUdKKKu5QTgT7b/0ojjVsUiiw309bc1LuAzoFf3TpsunBGTpMOQtBQREn1dUm1M+Uhi6NV7CrGuS4APWsEpUchkT084UddLukDSBYRU9nqSHm50W2MnsA6ROnkMeBB4AVjE9v9JutkNmk0yJmwPlXR6ubmEpHmBfwG3d2GnZNSW4GmB9xQD7s4lOnLuJRznyos2634/fYHdJQ0BLrB9bflNLUKkQPdtoFOyBRH928L22bXttq8rhetXS/o5MXTyzUbYUAVV5+w6ioyYJB1GqXTvQ1Tcr0A4Jb+phZUV0tMz2n6yOivHTt1JtR/RaTMvMb32CNsvlvz9Z0SHQ8PmdzSCsrAdSOg2PFIKWpcm6jaGEq3QvYgIyhGucN6MpKkIDZVFibbglnqvO5ry2e1FzI06QSGatgqRFr0R+KKZ6rPK72dHYqDldoRu0f7AG0S9xyS2P2jg8ackorNXAYfaPqNs70bMxTmFkJ2/tlE2dDaLLDbQNzQwYtK/EyMmWfyadBi2R5QrtnOJtsBPgR9I6i+pl+0Pmtwpmb0uUrIrIVI1iJD2nh/A9nbElV4rLpRbEc7iUKI2YWqilfQdwiHZlHAqT6jSKQFwqMxeAazbou91R7MfUavVX9LUjom3jxPpNzWLU6KgG1ET9ARxHnis3H0a4Vx90iinpFwcQYwpmBk4FdhH0vYQtWwlyrlTieC0S5ABaJ924XRMkg7HMYfjLOAuYu7EsZS5Hc1IOZkeDZwm6adE4W5tqukKhEbE1Ao59Nqi2TJI6ibpHOAN2/sTRab3AJ+XQtKdiUXjfeIqslmG4F3tFlPe7ChGs2CeCvyGuNpfsRQq3w1sbPupzrZvVGr2lmLXkcDfgNeI9v9DiPSngYsblWYq0c6a4NzRwHdtDyZa+7eXtF8pDsZFZK7KlFcyZrLGJGkIjmFr1xHtn6e7Sds7ywl1K6K+Yn/iSmte4Fmi4HUwsWDPSrQ3tiK7EgMFuxFRiBWJQsSfS7qbaJe8y/ZZlVk4GrrqojFKoevexIL+JnAmMDnh7G8K/Lq++6Yq9JVicF9ibtHrwJO2b5J0jKRehLDZVm6QwJuk3YDPJJ3ikJt/j4goYftxSX8mfstNPRz029H5bb2NIiMmSSN5CdiynKCa9RfzB6Jo8JwSCVmdaGuel3BE1gN+R0QSWjWlcAYh4z6rpPuASW2vQLyu24guj2cqtC+po84hG0QUtQ4hZrn8lIh0PQH8uRmcEvhSMXhaYHvC3p7EpPBZiblYnxNTghvllMxKpG1mBH5dzjXDgT9IWkjSEoR20g9tt4MIYtuTEZOkYdj+jAjnNvPV72e2LwEoV3YvEJ0qVxKS57sBI20Prc7EiUNfjZYfbvtTSesSYwBWB860fWt53OUu+hhJcyBpbsKBvEHSLsQQPoiIyRGlBbfqluBu/kp/aCDh3F5NpEBPJlIpN0i6zQ3SKSo2vCRpL8IJWg741PYhpYZqJ6IIdj/H8M22RXR+LUijSMck6ZKUq6pewAaSHrR9HdE58JDtzyUdRYjAvdvETtUYkdTT9seSZiKuHA8rJ/B1CHXQB4ETifRAy72+dmM0TsZHwBaSvkd0t3wOHAK87KJf0gQtwSMlzUDMt5qJUAuejZi2PSvwM0l3NrLl3F9Nwv4joVL8NrCYpI+BA4jv9uSubnBhMhFkKifpkpQivQ+ItsF1JC1etn2u0GNZBDij1ZySUsh7DLBxuWJcioiYvARQwumDCZnwW0snVVOq7nYVylW/y2e3s6Q1gL7A1sTnty7RzXKCv5p9UxklElfTKTmBSKMcBfyXKNBdjKhpOraRTkmxpRuwODC17YsJR/s/RN3YKUCPdEpaj3RMkq7O5cC7wF6SNpe0EvBj4LBmWAQmghmIq+2ZiW6EN4CXJc0hqU9JEQwFfmw760qagHLVX5uq/ClR87Mp8DRRqPwM4SQ3RSqiFJfOSDgj19geXJz6dYkoXG/grEbVZEnaXdJAxeTrkcAnhODhyiV9/BhRgHuym3AgaDJuMpWTdGlsv6GYGLwksQhATGWtZDz8t6W0+u6nEOBaisivb0XMKdkOeBL4pe23qrMyga/PviG+e+/bPl3SIKIVuJ/tu4iZQZVSHKftgKmIKMkGhBO8rkIh+FkA28c32I5JiPb2HwJzS9qV+E7vApxZUrCLAoNd5nB1JbLGJEnaBIeY2HXlX0tS12La0/bHtq8qLZNbACcRg/juBB5Np6R6RtHcWJZwHHcv6ZEziHPzLpIedQMVUieAPYiarA+A6Yg6klOJuUY7Srqe+G691CgDSsprRNEceoBwjLYm6llOISI4XxAdS13OKWkn0jFJkhanzinpAxwo6TPgO4S0/J+Bg4EFga1tf1KhqQnfmH3zN0JOfvMS5foNoTWzHpEOqdQpKTbuATxj+/JR7v5I0ilEB85viShKw6irhbqDGDB5HlHPcinR9t8DWN/2A420o5lpFx2TdEySpMUpTkkv4uR8KdGivQeh5roloS1xfjolzUGdU7I/cL/tY8td2xFpihWBa20/WJWNdcwNfFRzSuq7hyRtTUR3biGGLHZKJM72g5K+S9TkXGv7VkmvElL393eGDUljScckSdqDaQmJ+Zo2yRHEEMKPgUNSp6R6RtH92JYYwveUYkbTCyU6cnDV+iSjMDMh7HZyud0NGFE6vmYvOjlnVWTbl993Qrn4i4rsaA4qmGnTKLIrJ0lakJqSbp2i7sfEPJAtyu3+xKTnqdIpqZ463Q9JWg34JyHk9ziwhqQlJX0Hmk6M8G7gHUlr1upiyvaFgb6Spqj7DnYati8CJpO0Y7ndtZ0SisBaA/91JhkxSZIWQ1+fTfI7Se8DjwA/Aa6TNDvRsvmHqmsUkm9ESn5LOIyf2b5G0qXEfJnViLRb5YzSLfQ5oYa8KhGVO00h8b40cHCjdUrGYJ+A7kQRbssWrCdjJh2TJGkxilMyDdGR8AgwEjgM2I+Yijw1oVib3TdNQPm8ugH7AI/YvqJEThYkhkU+TNQAPVepocTcmaIQPD3wM8IpOZTo7vqppMWIzpzjG6VTMi5KROkzSbvZ/rgKG5qWNknlpGOSJC1AuUrckZjf8xShTPua7QskTQaMABazfSUhqpZUTEnZfGj75uKc1NptawXL8xBX/qc1QypC0rHAJJIuBH4NvAOsDSxaCnRPK4/rbvvz6iwN0ilpX7LGJEmanOKU7AksARxJFCP2I7oSasMSRxI5//xNNwGSasWty0kaIKkHMeF2VUkLS5qFGHz3fpM4JVMQk4ufJCJx/7W9F/EdW1LSaSXCA6EVkjQhauB/nUlGTJKkiSlOyeHAfbYPl7Q58Atikmo/STcS4lLzAUfm3JvqkbQdMZ9ou3J7EkJj42DgaCItMiUxS6YZ0jcqtSLnF6n5vsBASQvZfkTSP4ghee9B0xXnJm1IOiZJ0twcA6xMKF1CFCN2J65q7yUWi+HELJVXK7Ew+ZJSeLwksHm5vSewHDAvcK7tLUr0ZIDtV6qzNKgT55uGGLx3KSH6ti7wc0nTEsP5VrH9eoWmJuNBu7QLp2OSJE1KqR25DHgeWFHSD4kQ+/qlI2cvYrF7pEo7k6+w/UKJkLwi6VpCbn57YrDisZJOK7LtlTsl8GWtSx/CxveB+W3fLul8IlK3ErBWdnclnUk6JknSpJTakVslPUK0kq7EV5GT7xB1JjnSvQkYpSV4e2AzItKwj+03Jc1KdOA0RcFmXct5d+BA4HLbN5duoRWAh4CrgAfSKWkd2iRgko5JkjQ7todKOr3cXFTSYOBtQqfkzQpNSwp1LcE9bX8IHA8g6bslHbIqcILtIVXaWWyqib31I9KAbxKFubUIygxE9OTirCdJqiAdkyRpAWy/K+lkouZkcWCDdEqqR9L6RFThGeBM4ENJr9g+QtJ0RC3QQsBezTD7Rl8NEOwHXADcBAwA1pL0OeGoLADckE5JC9ImIZN0TJKkRbD9kaR/ErNvGjZePhk/JP0E2BRYsERL7gVuBg6TtAtwItGFM4XtF6qz9CuKUzKAKHT9U1GfnQbYl2hv7g0cmOJ8SZWkY5IkrcXVeSVbPaWN+0bgJUIhdQXgEtuPS9qGKFpeBtjGdtMI3kmairCtpqOC7Xck7WX7M0lT2f6oUiOTiaaz9Ua+cXxpJWAt4C0iM3jAxOwnHZMkaSHSKameukLXEZJeAC4EpgLWl/SJ7SeKvszNtt+r1Fi+UZg7AvgNsBNwNiEz360UWkOTFOcmrYekKYG/EJ1dwyVdImlF2zdO6L5SJTJJkmQ8qetmkaR9ifk3HwCXEF03v5e0FjDY9o1VTN4dxd5aoes0kpYHBtp+yvaOREvzs8QgyO6Qjm8rI0LHpFH/xoOlgZdsDy+37yTEICeYjJgkSZKMJzWnBNiKaAd+yPYnkp4ihO9+DQyptdhWvdCXLpt+RAvzO4Ro2vW2/wzsAZxDtApXPvsm+XY88MD9107RXdM28BCTS7qv7vaptk+tuz0A+LDu9gdl2wSTjkmSJMk4qHWzlJuLE4PtapLzUwMbAOcS3VJNIZ4GUFRm/wT8ndAmmYMQ65sLOB3YMHVK2gPbP6vYhLeAXnW3e5dtE0ymcpIkScZCSYeMkNStpG+mBL6QtGh5yKTEfJmRzeCU1KePSlj9NEJ5dg9iXs9ZwA+Bt9MpSTqQu4FZizMM8APg6onZUUZMkiRJxsAokZK9gWmIcPXUwNaSngMmA66xXXnhaN3sm97AwsBrtu+UNB8xZ+nHwOzAqjlbKelISkpzO+A4SW8Dj0xM4SuAstYpSZJkzBSNkr8DN9o+s2zrTQxX7A08bPuBseyiUymppd8RDtRmwM7AJ0T9y4zADrZfrMq+JBkX6ZgkSZKMBUnrAAcRYwCWr2u9bQpK6mYN4HbbQyQdCVxg+/6iFnwHcFm5ou3ZDJGdJBkb6ZgkSZLUUUuHjGb7GUQtye7AMNuvd7pxo1CckgOJoY4v2D5I0qFE2uZj4CSinmRO4KRmc6qSZHRkjUmSJEmhXqeEiJJ8RDghx9r+jaQrgX8TtRqVOibFxj8Cj9ner27bu8CCwDOEwutCwNnplCStQjomSZIkhTqnZE+iy+B9YLCkz22fCBwLfGr7sSrtLCwFdLd9AYCkY4ii3NWBy4m6km0IsbfXKrMySSaQdEySJEm+zveBnsA9wPqEDsgekpYAdrf9bpXG1TEE2EXS/MBIYFZgB+BlYPES4ZnC9qdVGpkkE0o6JkmSdGnqa0pKe/A9kt4kZso8DlxLaDJc1UROCbafKROOVyBUaK8qrcIjgJ5FZn5YpUYmyUSQxa9JknRZajolJX2zNbAAcDFwG7AbMA8wOZEOeXRMhbHNgKTpgBmAXwLn2H6hYpOSZKLIiEmSJF0SSd1tfy5pUmA7QrJ9UkID5HXgPqLG5Cnbj0L1s2/GhKQFgOOJNM7eWVOStDIZMUmSpMshaWeihXYQMC+wCbArUVPyATAJcH0rSbYXddf30ilJWp2clZMkSZdC0sbEsLGbibk3zxOy8mcQtRpXEoP6JmoyalXYfjydkqQdyIhJkiRdBkmbAZPb/ku5vSBRV/JjYDjwGtCdUEp9pCo7k6QrkzUmSZJ0CSRNQjghT0rqS9SSrA08SRSMbgm8A7yUTkmSVEc6JkmSdAlK982VwI3AqkAPYDXbT0nan6jP+GelRiZJko5JkiRdB9u3SpodeA/43PYwSXMDMwGXVWtdkiSQNSZJknRRJK1NFLj2By7P9E2SNAfZlZMkSVflPUI19dp0SpKkeciISZIkXRZJPW1/XLUdSZJ8RTomSZIkSZI0DZnKSZIkSZKkaUjHJEmSJEmSpiEdkyRJkiRJmoZ0TJIkSZIkaRrSMUmS5Fsh6fuSbpF0l6RBkk6UdLykiT6/SNpe0ot1t+8rkvJje86uE3iMX0h6QdJso2xfQNJNZa7OmJ47s6RLJA2agON97TUlSTJ60jFJkuRbYfte4BbgLtuDbO8ALAT87Fvs86RRNi1he8Q4njZBjontq4GXRrP9UeC2cTz3FWIK8YQcb9TXlCTJaEhJ+iRJOhRJkwLTAkMkHQH8CjgTWAp4AjgSOAx4FJgLOMX2/UUq/jjgAeDVuv2tBhwnaXnbL0paHfgp8CKwNLA7sDIwdYlg/Bu4ATgWeAvoAzxk+2xJPYDTgTeB18t943o92wHzA28AswHb2v6i3D2fpN8BCwKP2z68vP5vHHuC3sQk6cKkjkmSJN+a4hD8BLgOmAa42/Z55b5hwAzAB8QCvidwpe3zShrlMtuLSvoHcLHtCyXNBdxge7ayj1uAzYD3gf8Cc9oeLml54BXbz0l6se7x2wADbW8tSYRD9CNgXWAB29uVVNOLwI9svzia1/Oi7bMk/RK42vZISccRSrFXl1TPSrY3Ls95HFgP+MHojm37rXobkyQZPRkxSZKko7jL9qDRbH/T9tDy90OSFgLekjQLoPJ3NyIqcVh53PNjOMZcwLu2hwPYvmUMj1sImEHSXuX2o8D05RjPlOeOHM+aj0+AwZKGAPMREZ0a9XY+V+4f07HfGo9jJUmXJx2TJEkazahh2YeBG21fUSIKrxYn4XFgbmLhn2MM+3oW6CdpMtuflYjJG7afBEYCSFq4HGO47cPLtjWJ6MjjwAJlWzciNTMuLgYWtv2ypN6j3Fdv51xl/1OP4dhJkowHmcpJkuRbIWlxYDAwGXCM7Uvq7tuSqCk5yPafyrYZgQOJCEM/Iu1zqaQ5geMJp+I9YC9gx/L3CcCFtvcqNSarEIWr/YB9bH9eUkHPAh8BhxebPgK+ALB9kKTJgTOAIcC7ROrlKtu/r7N5AaLWZSiwc7FhAeAOoqD3XaLQ9lhgCuAeYB7gUduHlO6h0R17e+AQYHvb53+rNz1J2ph0TJIkSZIkaRqyXThJkiRJkqYhHZMkSZIkSZqGdEySJEmSJGka0jFJkiRJkqRpSMckSZIkSZKmIR2TJEmSJEmahnRMkiRJkiRpGtIxSZIkSZKkafh/HjttJG9rTe4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Evaluate for each epcoh\n",
    "eval_examples = processor.get_dev_examples(data_dir)\n",
    "eval_labeled_examples = []\n",
    "for example in eval_examples:\n",
    "    if example.label not in label_unknown:\n",
    "        eval_labeled_examples.append(example)\n",
    "        \n",
    "eval_features = convert_examples_to_features(eval_labeled_examples, label_list, max_seq_length, tokenizer)\n",
    "logger.info(\"\")\n",
    "logger.info(\"***** Running evaluation *****\")\n",
    "logger.info(\"  Num examples = %d\", len(eval_features))\n",
    "logger.info(\"  Batch size = %d\", eval_batch_size)\n",
    "eval_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)\n",
    "eval_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)\n",
    "eval_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)\n",
    "eval_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)\n",
    "eval_data = TensorDataset(eval_input_ids, eval_input_mask, eval_segment_ids, eval_label_ids)\n",
    "# Run prediction for full data\n",
    "eval_sampler = SequentialSampler(eval_data)\n",
    "eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=eval_batch_size)\n",
    "\n",
    "\n",
    "global_step = 0\n",
    "u = 0.95\n",
    "l = 0.455 \n",
    "eta = 0\n",
    "\n",
    "y_pred_last = np.zeros_like(eval_label_ids)\n",
    "for _ in trange(int(num_train_epochs), desc=\"Epoch\"):\n",
    "    model.train()\n",
    "\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(tqdm(train_labeled_dataloader, desc=\"Iteration (labeled)\")):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        input_ids, input_mask, segment_ids, label_ids = batch\n",
    "        loss = model(input_ids, segment_ids, input_mask, u, l, 'train', label_ids)\n",
    "        loss.backward()\n",
    "\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        global_step += 1\n",
    "    train_labeled_loss = tr_loss / nb_tr_steps\n",
    "    \n",
    "    \n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(tqdm(train_semi_dataloader, desc=\"Iteration (all train)\")):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        input_ids, input_mask, segment_ids, label_ids = batch\n",
    "        loss = model(input_ids, segment_ids, input_mask, u, l, 'train', label_ids, True)\n",
    "        loss.backward()\n",
    "\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        global_step += 1\n",
    "    train_loss = tr_loss / nb_tr_steps\n",
    "    \n",
    "    \n",
    "    \n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "\n",
    "    y_preds = []\n",
    "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        input_ids, input_mask, segment_ids, label_ids = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            tmp_eval_loss = model(input_ids, segment_ids, input_mask, u, l, 'train', label_ids)\n",
    "            logits = model(input_ids, segment_ids, input_mask)\n",
    "\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = label_ids.to('cpu').numpy()\n",
    "\n",
    "        eval_loss += tmp_eval_loss.mean().item()\n",
    "        y_preds.append(np.argmax(logits, 1))\n",
    "        nb_eval_steps += 1\n",
    "\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    y_pred = np.hstack(y_preds)\n",
    "    y_eval = eval_label_ids.numpy()\n",
    "\n",
    "    results = clustering_score(y_eval, y_pred)\n",
    "    ## Confusion matrix(reorder y_pred for alignment)\n",
    "    ind, w = hungray_aligment(y_eval, y_pred)\n",
    "    d_ind = {i[0]: i[1] for i in ind}\n",
    "    y_pred_ = pd.Series(y_pred).map(d_ind)\n",
    "    cm = confusion_matrix(y_eval, y_pred_)\n",
    "\n",
    "    delta_label = np.sum(y_pred != y_pred_last).astype(np.float32) / y_pred.shape[0]\n",
    "    y_pred_last = np.copy(y_pred)\n",
    "\n",
    "    results['u'] = u\n",
    "    results['l'] = l\n",
    "    loss = tr_loss\n",
    "    result = {'eval_loss': eval_loss,\n",
    "              'results': results,\n",
    "              'global_step': global_step,\n",
    "              'train_labeled_loss': train_labeled_loss,\n",
    "              'train_loss': train_loss,\n",
    "              'delta_label': delta_label}\n",
    "    print(cm)\n",
    "    output_eval_file = os.path.join(output_dir, \"eval_results.txt\")\n",
    "    with open(output_eval_file, \"w\") as writer:\n",
    "        logger.info(\"***** Eval results *****\")\n",
    "        for key in sorted(result.keys()):\n",
    "            logger.info(\"  %s = %s\", key, str(result[key]))\n",
    "            writer.write(\"%s = %s\\n\" % (key, str(result[key])))\n",
    "    print(f\"{_}: loss={train_loss}, (u, l) = ({round(u, 4)},{round(l, 4)})\")\n",
    "    eta += 1.1 * 0.009\n",
    "    u = 0.95 - eta\n",
    "    l = 0.455 + eta*0.1\n",
    "    if u < l:\n",
    "        break\n",
    "results_all.update({'CDAC': results})\n",
    "print(results, label_unknown)\n",
    "\n",
    "plot_confusion_matrix(cm, label_list, normalize=False, figsize=(8, 8),\n",
    "                      title='Confusion matrix, accuracy=' + str(results['ACC']))\n",
    "# Save a trained model and the associated configuration\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self\n",
    "output_model_file = os.path.join(output_dir, WEIGHTS_NAME)\n",
    "torch.save(model_to_save.state_dict(), output_model_file)\n",
    "output_config_file = os.path.join(output_dir, CONFIG_NAME)\n",
    "with open(output_config_file, 'w') as f:\n",
    "    f.write(model_to_save.config.to_json_string())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec8598cf9e3f4556af56813969c02d47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Extracting representation I', max=52, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fc627fd2e66440ba44e0c1eb08872c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Extracting probabilities Q', max=3, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ACC': 92.43, 'ARI': 83.4, 'NMI': 86.52}\n"
     ]
    }
   ],
   "source": [
    "test_examples = processor.get_test_examples(data_dir)\n",
    "test_features = convert_examples_to_features(test_examples, label_list, max_seq_length, tokenizer)\n",
    "logger.info(\"\")\n",
    "logger.info(\"***** Running testuation *****\")\n",
    "logger.info(\"  Num examples = %d\", len(test_examples))\n",
    "logger.info(\"  Batch size = %d\", eval_batch_size)\n",
    "test_input_ids = torch.tensor([f.input_ids for f in test_features], dtype=torch.long)\n",
    "test_input_mask = torch.tensor([f.input_mask for f in test_features], dtype=torch.long)\n",
    "test_segment_ids = torch.tensor([f.segment_ids for f in test_features], dtype=torch.long)\n",
    "test_label_ids = torch.tensor([f.label_id for f in test_features], dtype=torch.long)\n",
    "test_data = TensorDataset(test_input_ids, test_input_mask, test_segment_ids, test_label_ids)\n",
    "test_sampler = SequentialSampler(test_data)\n",
    "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=eval_batch_size)\n",
    "\n",
    "k = num_labels*fraction\n",
    "y_true = train_label_ids\n",
    "num_train_epochs = 100\n",
    "learning_rate = 5e-5\n",
    "\n",
    "# Initialize cluster centroids U with representation I\n",
    "embs_train = []\n",
    "for batch in tqdm(train_dataloader, desc=\"Extracting representation I\"):\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    input_ids, input_mask, segment_ids, label_ids = batch\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        logits, q = model(input_ids, segment_ids, input_mask, mode='finetune')\n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    label_ids = label_ids.to('cpu').numpy()\n",
    "    embs_train.append(logits)\n",
    "emb_train = np.vstack(embs_train)\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "km = KMeans(n_clusters=k, n_jobs=-1, random_state=seed)\n",
    "km.fit(emb_train)\n",
    "y_pred_last = np.copy(km.cluster_centers_)\n",
    "model.cluster_layer.data = torch.tensor(km.cluster_centers_).to(device)\n",
    "\n",
    "model.eval()\n",
    "\n",
    "# Extracting probabilities Q\n",
    "embs_test = []\n",
    "qs = []\n",
    "for batch in tqdm(test_dataloader, desc=\"Extracting probabilities Q\"):\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    input_ids, input_mask, segment_ids, label_ids = batch\n",
    "    with torch.no_grad():\n",
    "        logits, q = model(input_ids, segment_ids, input_mask, mode='finetune')\n",
    "    q = q.detach().cpu().numpy()\n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    qs.append(q)\n",
    "    embs_test.append(logits)\n",
    "\n",
    "q_all = np.vstack(qs)\n",
    "y_pred = q_all.argmax(1)\n",
    "results = clustering_score(test_label_ids, y_pred)\n",
    "results_all.update({'CDAC-KM': results})\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tony/anaconda3/envs/BERT/lib/python3.6/site-packages/ipykernel_launcher.py:45: DeprecationWarning: elementwise != comparison failed; this will raise an error in the future.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 {'ACC': 93.55, 'ARI': 85.48, 'NMI': 87.3, 'kl_loss': 0.0392, 'delta_label': 0.0001}\n",
      "1 {'ACC': 95.68, 'ARI': 90.64, 'NMI': 90.91, 'kl_loss': 0.0377, 'delta_label': 0.0313}\n",
      "2 {'ACC': 96.15, 'ARI': 91.69, 'NMI': 91.76, 'kl_loss': 0.0386, 'delta_label': 0.0099}\n",
      "3 {'ACC': 96.29, 'ARI': 92.04, 'NMI': 92.12, 'kl_loss': 0.0393, 'delta_label': 0.0062}\n",
      "4 {'ACC': 96.33, 'ARI': 92.15, 'NMI': 92.3, 'kl_loss': 0.0398, 'delta_label': 0.0035}\n",
      "5 {'ACC': 96.33, 'ARI': 92.17, 'NMI': 92.39, 'kl_loss': 0.0399, 'delta_label': 0.0034}\n",
      "6 {'ACC': 96.32, 'ARI': 92.14, 'NMI': 92.38, 'kl_loss': 0.0397, 'delta_label': 0.0033}\n",
      "7 {'ACC': 96.26, 'ARI': 92.05, 'NMI': 92.35, 'kl_loss': 0.04, 'delta_label': 0.0019}\n",
      "8 {'ACC': 96.25, 'ARI': 92.02, 'NMI': 92.37, 'kl_loss': 0.0399, 'delta_label': 0.002}\n",
      "9 {'ACC': 96.19, 'ARI': 91.91, 'NMI': 92.31, 'kl_loss': 0.0398, 'delta_label': 0.0014}\n",
      "10 {'ACC': 96.16, 'ARI': 91.85, 'NMI': 92.25, 'kl_loss': 0.04, 'delta_label': 0.0016}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c977c84eaf3479782fb9a420c8446db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Extracting probabilities Q', max=3, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ACC': 94.0, 'ARI': 87.58, 'NMI': 89.87}\n"
     ]
    }
   ],
   "source": [
    "# Prepare optimizer\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "]\n",
    "num_train_optimization_steps = int(len(train_examples) / train_batch_size) * num_train_epochs\n",
    "optimizer = BertAdam(optimizer_grouped_parameters, lr=learning_rate, warmup=warmup_proportion, t_total=num_train_optimization_steps)\n",
    "\n",
    "\n",
    "import copy\n",
    "model_best = None\n",
    "nmi_best = 0\n",
    "wait, patient = 0, 5\n",
    "for epoch in range(num_train_epochs):\n",
    "    # Calculate probabilities P (as target)\n",
    "    model.eval()\n",
    "    qs = []\n",
    "    for batch in train_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        input_ids, input_mask, segment_ids, label_ids = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            logits, q = model(input_ids, segment_ids, input_mask, mode='finetune')\n",
    "        q = q.detach().cpu().numpy()\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        qs.append(q)\n",
    "    q_all = np.vstack(qs)\n",
    "    p_all = target_distribution(q_all)\n",
    "    y_pred = q_all.argmax(1)\n",
    "    results = clustering_score(y_true, y_pred)\n",
    "    \n",
    "    # early stop\n",
    "    if results['NMI'] > nmi_best:\n",
    "        model_best = copy.deepcopy(model)\n",
    "        wait = 0\n",
    "        nmi_best = results['NMI']\n",
    "    else:\n",
    "        wait += 1\n",
    "        if wait > patient:\n",
    "            model = model_best\n",
    "            break\n",
    "    \n",
    "    delta_label = np.sum(y_pred != y_pred_last).astype(np.float32) / y_pred.shape[0]\n",
    "    y_pred_last = np.copy(y_pred)\n",
    "    if epoch > 0 and delta_label < 0.001:\n",
    "        print(epoch, delta_label, 'break')\n",
    "        break\n",
    "\n",
    "    # Fine-tuning with auxiliary distribution\n",
    "    model.train()\n",
    "    tr_loss, nb_tr_examples, nb_tr_steps = 0, 0, 0\n",
    "    qs = []\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        input_ids, input_mask, segment_ids, label_ids = batch\n",
    "        logits, q = model(input_ids, segment_ids, input_mask, mode='finetune')\n",
    "        kl_loss = F.kl_div(q.log(), torch.Tensor(p_all[step*train_batch_size: (step+1)*train_batch_size]).cuda())\n",
    "        kl_loss.backward()\n",
    "\n",
    "        tr_loss += kl_loss.item()\n",
    "        nb_tr_examples += input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()        \n",
    "    train_loss = tr_loss / nb_tr_steps\n",
    "    results['kl_loss'] = round(train_loss, 4)\n",
    "    results['delta_label'] = delta_label.round(4)\n",
    "    print(epoch, results)\n",
    "\n",
    "\n",
    "# Extracting probabilities Q\n",
    "embs_test = []\n",
    "qs = []\n",
    "for batch in tqdm(test_dataloader, desc=\"Extracting probabilities Q\"):\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    input_ids, input_mask, segment_ids, label_ids = batch\n",
    "    with torch.no_grad():\n",
    "        logits, q = model(input_ids, segment_ids, input_mask, mode='finetune')\n",
    "    q = q.detach().cpu().numpy()\n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    qs.append(q)\n",
    "    embs_test.append(logits)\n",
    "\n",
    "q_all = np.vstack(qs)\n",
    "y_pred = q_all.argmax(1)\n",
    "y_true = test_label_ids\n",
    "\n",
    "results = clustering_score(y_true, y_pred)\n",
    "results_all.update({'CDAC+': results})\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "BERT",
   "language": "python",
   "name": "bert"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
